diff --git a/library/src/amd_detail/rocblaslt/include/Debug.hpp b/library/src/amd_detail/rocblaslt/include/Debug.hpp index 9bdb0c1f41..4330a0e6e5 100644 --- a/library/src/amd_detail/rocblaslt/include/Debug.hpp +++ b/library/src/amd_detail/rocblaslt/include/Debug.hpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/library/src/amd_detail/rocblaslt/include/rocblaslt-types.h b/library/src/amd_detail/rocblaslt/include/rocblaslt-types.h index 47c835b831..f040b6c4c0 100644 --- a/library/src/amd_detail/rocblaslt/include/rocblaslt-types.h +++ b/library/src/amd_detail/rocblaslt/include/rocblaslt-types.h @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. + * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -389,8 +389,8 @@ typedef struct __attribute__((packed, aligned(8))) _rocblaslt_matmul_algo typedef struct _rocblaslt_matmul_algo{ #ifdef __cplusplus uint8_t data[8] = {0}; // must match hipblasLtMatmulAlgo_t layout - bool fallback = false; // - uint8_t data_pad[7] = {0}; // has uint8_t data[16] + bool fallback = false; // + uint8_t data_pad[7] = {0}; // has uint8_t data[16] size_t max_workspace_bytes = 0; #else uint8_t data[8]; diff --git a/library/src/amd_detail/rocblaslt/src/CMakeLists.txt b/library/src/amd_detail/rocblaslt/src/CMakeLists.txt index e9846346dc..03549309bd 100644 --- a/library/src/amd_detail/rocblaslt/src/CMakeLists.txt +++ b/library/src/amd_detail/rocblaslt/src/CMakeLists.txt @@ -122,7 +122,7 @@ string(REGEX MATCH "([0-9]+\\.[0-9]+\\.[0-9]+)" GCC_VERSION "${GCC_OUTPUT}") message(STATUS "Extracted GCC Version: ${GCC_VERSION}") # Compare GCC version and set the flag conditionally if it's 7.5.0 or less -if(${GCC_VERSION} VERSION_LESS "7.6.0") +if(${GCC_VERSION} VERSION_LESS "7.6.0") # Apply -ftemplate-depth=2048 to tensile_host.cpp if GCC_VERSION is less than 7.6.0 to avoid template recursion set_source_files_properties(src/amd_detail/rocblaslt/src/tensile_host.cpp PROPERTIES COMPILE_FLAGS "-ftemplate-depth=2048") endif() diff --git a/library/src/amd_detail/rocblaslt/src/Debug.cpp b/library/src/amd_detail/rocblaslt/src/Debug.cpp index bd0730763b..a867107943 100644 --- a/library/src/amd_detail/rocblaslt/src/Debug.cpp +++ b/library/src/amd_detail/rocblaslt/src/Debug.cpp @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -60,7 +60,7 @@ namespace rocblaslt #ifndef HIPBLASLT_ENABLE_MARKER if(m_printMarker) printf("HIPBLASLT_ENABLE_MARKER is not defined. Please rebuild with -DHIPBLASLT_ENABLE_MARKER=ON\n"); -#endif +#endif } const char *hipblaslt_preload = std::getenv("HIPBLASLT_PRELOAD_KERNELS"); diff --git a/library/src/amd_detail/rocblaslt/src/include/logging.h b/library/src/amd_detail/rocblaslt/src/include/logging.h index ffc32cfbaa..42c16ad5a2 100644 --- a/library/src/amd_detail/rocblaslt/src/include/logging.h +++ b/library/src/amd_detail/rocblaslt/src/include/logging.h @@ -3,7 +3,7 @@ * * MIT License * - * Copyright (C) 2022 Advanced Micro Devices, Inc. + * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/library/src/amd_detail/rocblaslt/src/include/utility.hpp b/library/src/amd_detail/rocblaslt/src/include/utility.hpp index 9f85ed9ddd..253b9340d9 100644 --- a/library/src/amd_detail/rocblaslt/src/include/utility.hpp +++ b/library/src/amd_detail/rocblaslt/src/include/utility.hpp @@ -3,7 +3,7 @@ * * MIT License * - * Copyright (C) 2022-2024 Advanced Micro Devices, Inc. + * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/tensilelite/Tensile/Activation.py b/tensilelite/Tensile/Activation.py index 67f0f6d2c8..6b92561af6 100644 --- a/tensilelite/Tensile/Activation.py +++ b/tensilelite/Tensile/Activation.py @@ -31,7 +31,7 @@ TensileInstructions from .TensileInstructions.Enums import * from .TensileInstructions.Instructions import * -from .Common import printExit, printWarning +from Tensile.Common.Utilities import printExit, printWarning from dataclasses import dataclass, field diff --git a/tensilelite/Tensile/BenchmarkProblems.py b/tensilelite/Tensile/BenchmarkProblems.py index 350ab1fa56..40413b8cb0 100644 --- a/tensilelite/Tensile/BenchmarkProblems.py +++ b/tensilelite/Tensile/BenchmarkProblems.py @@ -30,57 +30,131 @@ from copy import deepcopy from pathlib import Path +from typing import Dict + +from Tensile import CUSTOM_KERNEL_PATH, ClientExecutable, SolutionLibrary, LibraryIO +from Tensile.TensileLogic.ValidMatrixInstruction import validateMIParameters +from Tensile.Toolchain.Component import Assembler +from Tensile.SolutionStructs.Problem import ProblemType, ProblemSizes +from Tensile.SolutionStructs import Solution, matrixInstructionToMIParameters +from Tensile.SolutionStructs.Naming import getMinNaming, getNameMin, getSerialNaming, getNameFull, \ + getKeyNoInternalArgs -from . import CUSTOM_KERNEL_PATH, ClientExecutable, SolutionLibrary, LibraryIO from .BenchmarkStructs import BenchmarkProcess, constructForkPermutations from .Contractions import ProblemType as ContractionsProblemType from .ClientWriter import runClient, writeClientConfig, writeClientConfigIni from .KernelWriterAssembly import KernelWriterAssembly -from .SolutionStructs import Solution, ProblemType, ProblemSizes +from .KernelWriter import DebugConfig from .TensileCreateLibrary import copyStaticFiles, writeSolutionsAndKernels from .CustomKernels import getCustomKernelConfig from .Toolchain.Assembly import AssemblyToolchain from .Toolchain.Source import SourceToolchain -from .Common import globalParameters, HR, print1, print2, \ - printExit, printWarning, ensurePath, startTime, tqdm, state, \ - BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR +from .Common import globalParameters, HR, print1, print2, IsaInfo, IsaVersion, \ + printExit, printWarning, ensurePath, startTime, tqdm, state, gfxToVariants, \ + BENCHMARK_PROBLEMS_DIR, BENCHMARK_DATA_DIR, isaToGfx, DepthUConfig -def generateForkedSolutions(problemType, constantParams, forkPermutations, cxxCompiler): +def _generateForkedSolutions(problemType, constantParams, forkPermutations, assembler: Assembler, \ + debugConfig: DebugConfig, depthUConfig: DepthUConfig, isaInfoMap: Dict[str, IsaInfo]): """Creates a list with a Solution object for each parameter combination in forkPermutations""" print1("# Enumerating Solutions") solutions = [] solutionSet = set() for perm in forkPermutations: - solution = {"ProblemType": deepcopy(problemType.state)} + # Expect only a single ISA in the map for the Tensile context + # because the GPU has to be physically present for benchmarking + isa = next(iter(isaInfoMap.keys())) + + solution = {} + solution.update({ + "ProblemType": deepcopy(problemType.state), + }) solution.update(constantParams) solution.update(perm) - # TODO check if solution matches problem size for exact tile kernels - solutionObject = Solution(solution, cxxCompiler) - if solutionObject["Valid"]: - if solutionObject not in solutionSet: - solutionSet.add(solutionObject) - solutions.append(solutionObject) - elif globalParameters["PrintSolutionRejectionReason"]: - print1("rejecting solution " + str(solutionObject)) + + mi = solution["MatrixInstruction"] + wavefrontSize = solution["WavefrontSize"] + workgroup = solution["WorkGroup"] + ptype = solution["ProblemType"] + + miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) + solution.update(miParams) + + if validateMIParameters(solution, isaInfoMap): + solutionObject = Solution( + solution, + debugConfig.splitGSU, + debugConfig.printSolutionRejectionReason, + debugConfig.printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap + ) + if solutionObject["Valid"]: + if solutionObject not in solutionSet: + solutionSet.add(solutionObject) + solutions.append(solutionObject) + elif debugConfig.printSolutionRejectionReason: + print1("rejecting solution " + str(solution)) return solutions -def getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler: str, directory=CUSTOM_KERNEL_PATH): +def _getCustomKernelSolutionObj( + kernelName, + internalSupportParams, + assembler: Assembler, + debugConfig: DebugConfig, + depthUConfig: DepthUConfig, + isaInfoMap: Dict[IsaVersion, IsaInfo], + directory=CUSTOM_KERNEL_PATH + ): """Creates the Solution object for a custom kernel""" - config = getCustomKernelConfig(kernelName, internalSupportParams, directory) - return Solution(config, cxxCompiler) - - -def generateCustomKernelSolutions(problemType, customKernels, internalSupportParams, failOnMismatch, cxxCompiler: str): + sol = getCustomKernelConfig(kernelName, internalSupportParams, directory) + + mi = sol["MatrixInstruction"] + isa = next(iter(isaInfoMap.keys())) + wavefrontSize = sol["WavefrontSize"] + ptype = sol["ProblemType"] + workgroup = sol.get("WorkGroup", None) + + # TODO: this should be deleted once all custom kernel configs MI are length 4. + if len(mi) == 9: + miParams = matrixInstructionToMIParameters(mi, isa, wavefrontSize, ptype, workgroup, isaInfoMap) + sol.update(miParams) + + sol = Solution( + sol, + debugConfig.printIndexAssignmentInfo, + debugConfig.printSolutionRejectionReason, + debugConfig.printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap + ) + + print1(f" --DBG-- wavefrontSize: {sol['WavefrontSize']}") + print1(f" --DBG-- workGroup: {sol['WorkGroup']}") + return sol + + +def _generateCustomKernelSolutions( + problemType, + customKernels, + internalSupportParams, + failOnMismatch, + assembler: Assembler, + debugConfig: DebugConfig, + depthUConfig: DepthUConfig, + isaInfoMap: Dict[str, IsaInfo] + ): """Creates a list with a Solution object for each name in customKernel""" solutions = [] for kernelName in customKernels: print1("# Processing custom kernel {}".format(kernelName)) - solution = getCustomKernelSolutionObj(kernelName, internalSupportParams, cxxCompiler) + solution = _getCustomKernelSolutionObj(kernelName, internalSupportParams, assembler, debugConfig, depthUConfig, isaInfoMap) # The ActivationType setting in YAML is meaningless in customKernel case. # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solution["ProblemType"]["ActivationType"] = problemType["ActivationType"] @@ -106,14 +180,30 @@ def generateCustomKernelSolutions(problemType, customKernels, internalSupportPar print1("# Added {} to solutions".format(kernelName)) if solution["Valid"]: solutions.append(solution) - elif globalParameters["PrintSolutionRejectionReason"]: + elif debugConfig.printSolutionRejectionReason: print1("rejecting solution " + str(solution)) return solutions -def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ - biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, stepName, solutionSummationSizes, \ - asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, sourcePath: Path): +def writeBenchmarkFiles( + stepBaseDir, + solutions, + problemSizes, + biasTypeArgs, + factorDimArgs, + activationArgs, + icacheFlushArgs, + stepName, + solutionSummationSizes, + asmToolchain: AssemblyToolchain, + srcToolchain: SourceToolchain, + sourcePath: Path, + useShortNames: bool, + debugConfig: DebugConfig, + depthUConfig: DepthUConfig, + deviceId: int, + isaInfoMap: Dict[IsaVersion, IsaInfo] + ): """Write all the files needed for a given benchmarking step""" ensurePath(sourcePath) @@ -128,7 +218,7 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ for solution in tqdm(solutions, "Finding unique solutions"): solutionKernels = solution.getKernels() for kernel in solutionKernels: - kName = Solution.getKeyNoInternalArgs(kernel) + kName = getKeyNoInternalArgs(kernel, debugConfig.splitGSU) if kName not in kernelNames: kernels.append(kernel) kernelNames.add(kName) @@ -140,23 +230,50 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ kernelHelperObjs.append(ko) kernelHelperNames.add(kname) - kernelSerialNaming = Solution.getSerialNaming(kernels) - kernelMinNaming = Solution.getMinNaming(kernels) - kernelWriterAssembly = KernelWriterAssembly(kernelMinNaming, kernelSerialNaming, asmToolchain.assembler, asmToolchain.assemblerVersion) - + kernelSerialNaming = getSerialNaming(kernels) + kernelMinNaming = getMinNaming(kernels) + kernelWriterAssembly = KernelWriterAssembly( + kernelMinNaming, + kernelSerialNaming, + asmToolchain.assembler, + debugConfig, + ) + + cmdLineArchs = [var for isa in isaInfoMap.keys() for var in gfxToVariants(isaToGfx(isa))] + # cmdLineArchs = [variant isaToGfx(isa) for isa in isaInfoMap.keys() for gfxToVariants()] # write solution, kernels and CMake problemType = solutions[0]["ProblemType"] codeObjectFiles, _= writeSolutionsAndKernels( \ - sourcePath, asmToolchain, srcToolchain, \ - solutions, kernels, kernelHelperObjs, \ - kernelWriterAssembly, errorTolerant=True, fromTensile=True, \ - generateSourcesAndExit=globalParameters["GenerateSourcesAndExit"]) + sourcePath, + asmToolchain, + srcToolchain, + solutions, + kernels, + kernelHelperObjs, + kernelWriterAssembly, + debugConfig.splitGSU, + cmdLineArchs, + kernelSerialNaming, + kernelMinNaming, + errorTolerant=True, + generateSourcesAndExit=globalParameters["GenerateSourcesAndExit"], # put in debug config + compress=False, + useShortNames=useShortNames + ) # ^ this is where solutions is mutated newLibraryDir = ensurePath(sourcePath / 'library') newLibraryFile = os.path.join(newLibraryDir, "TensileLibrary") - newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary(solutions, asmToolchain.assembler) - newLibrary.applyNaming(kernelMinNaming) + newLibrary = SolutionLibrary.MasterSolutionLibrary.BenchmarkingLibrary( + solutions, + asmToolchain.assembler, + debugConfig.splitGSU, + debugConfig.printSolutionRejectionReason, + debugConfig.printIndexAssignmentInfo, + depthUConfig, + isaInfoMap, + ) + newLibrary.applyNaming(debugConfig.splitGSU, kernelMinNaming) LibraryIO.write(newLibraryFile, state(newLibrary), globalParameters["LibraryFormat"]) codeObjectFiles = [os.path.relpath(f, sourcePath) \ @@ -186,11 +303,11 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ idealProblemSizes = ProblemSizes(problemType, idealSizes) writeClientConfig(True, solutions, idealProblemSizes, biasTypeArgs, \ factorDimArgs, activationArgs, icacheFlushArgs, stepName, stepBaseDir, \ - newLibrary, codeObjectFiles, True) + newLibrary, codeObjectFiles, True, deviceId) else: writeClientConfig(True, solutions, problemSizes, biasTypeArgs, \ factorDimArgs, activationArgs, icacheFlushArgs, stepName, stepBaseDir, \ - newLibrary, codeObjectFiles, False) + newLibrary, codeObjectFiles, False, deviceId) if len(solutions) == 0: printExit("write solutions and kernels results 0 valid soultion.") @@ -198,9 +315,11 @@ def writeBenchmarkFiles(stepBaseDir, solutions, problemSizes, \ return codeObjectFiles -def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache, +def _benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeGroupIdx, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, - buildTmpPath: Path, benchmarkProblemsPath: Path + buildTmpPath: Path, benchmarkProblemsPath: Path, useShortNames: bool, + debugConfig: DebugConfig, depthUConfig: DepthUConfig, deviceId: int, + isaInfoMap: Dict[str, IsaInfo] ): """Run the benchmarking for a single entry in the BenchmarkProblems of a Tensile config""" benchmarkTestFails = 0 @@ -210,7 +329,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG print1("# Converting Config to BenchmarkProcess Object") print1(HR) print1("") - benchmarkProcess = BenchmarkProcess(problemTypeConfig, problemSizeGroupConfig) + benchmarkProcess = BenchmarkProcess(problemTypeConfig, problemSizeGroupConfig, debugConfig.printIndexAssignmentInfo) enableTileSelection = benchmarkProcess.problemType["TileAwareSelection"] groupName = "{}_{:02d}".format(str(benchmarkProcess.problemType), problemSizeGroupIdx) @@ -280,11 +399,13 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG benchmarkStep.paramGroups) if problemSizeGroupConfig["ForkParameters"] else [] maxPossibleSolutions = len(forkPermutations) - regSolutions = generateForkedSolutions(benchmarkProcess.problemType, \ - benchmarkStep.constantParams, forkPermutations, srcToolchain.compiler) - kcSolutions = generateCustomKernelSolutions(benchmarkProcess.problemType, \ + regSolutions = _generateForkedSolutions(benchmarkProcess.problemType, \ + benchmarkStep.constantParams, forkPermutations, asmToolchain.assembler, \ + debugConfig, depthUConfig, isaInfoMap) + kcSolutions = _generateCustomKernelSolutions(benchmarkProcess.problemType, \ benchmarkStep.customKernels, benchmarkStep.internalSupportParams, \ - not benchmarkStep.customKernelWildcard, srcToolchain.compiler) + not benchmarkStep.customKernelWildcard, asmToolchain.assembler, debugConfig, \ + depthUConfig, isaInfoMap) maxPossibleSolutions += len(kcSolutions) solutions = regSolutions + kcSolutions @@ -295,7 +416,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG # handle no valid solutions if len(solutions) == 0: msg = "Your parameters resulted in 0 valid solutions." - if globalParameters["PrintSolutionRejectionReason"]: + if debugConfig.printSolutionRejectionReason: msg += "\nExamine reject and backtrace messages above to see why" \ "and where solutions were rejected." else: @@ -303,18 +424,17 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG "to see why each parameter combination was rejected." printExit(msg) - if globalParameters["PrintLevel"] >= 1: - for solution in solutions: - print2("# ({}:{}) {}".format(0, 0, Solution.getNameFull(solution))) - print2(HR) + for solution in solutions: + print2("# ({}:{}) {}".format(0, 0, getNameFull(solution, debugConfig.splitGSU))) + print2(HR) # write benchmarkFiles prevCount = len(solutions) codeObjectFiles = writeBenchmarkFiles(stepBaseDir, solutions, \ - benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, \ + benchmarkStep.problemSizes, benchmarkStep.biasTypeArgs, \ benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, \ benchmarkStep.icacheFlushArgs, shortName, [], asmToolchain, srcToolchain, \ - sourcePath) + sourcePath, useShortNames, debugConfig, depthUConfig, deviceId, isaInfoMap) # ^ this mutates solutions # write cache data @@ -332,17 +452,17 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG .format(len(solutions), prevCount )) # add SolutionIndex and SolutionNameMin into benchmark yaml - solutionMinNaming = Solution.getMinNaming(solutions) + solutionMinNaming = getMinNaming(solutions) for i in range(0, len(solutions)): solution = solutions[i] solution["SolutionIndex"] = i - solution["SolutionNameMin"] = Solution.getNameMin(solution, solutionMinNaming) - solution["KernelNameMin"] = Solution.getNameMin(solution, solutionMinNaming, True) + solution["SolutionNameMin"] = getNameMin(solution, solutionMinNaming, debugConfig.splitGSU) + solution["KernelNameMin"] = getNameMin(solution, solutionMinNaming, debugConfig.splitGSU, True) else: solutions = None print1("# Using cached solution data") - ssProblemType = ProblemType(problemTypeConfig) + ssProblemType = ProblemType(problemTypeConfig, debugConfig.printIndexAssignmentInfo) conProblemType = ContractionsProblemType.FromOriginalState(ssProblemType) outFile = os.path.join(sourcePath, "ClientParameters.ini") @@ -350,7 +470,7 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG benchmarkStep.factorDimArgs, benchmarkStep.activationArgs, benchmarkStep.icacheFlushArgs, conProblemType, stepBaseDir, codeObjectFiles, resultsFileName, - outFile) + outFile, deviceId) # I think the size portion of this yaml could be removed, # but for now it's needed, so we update it even in the cache case @@ -379,9 +499,22 @@ def benchmarkProblemType(problemTypeConfig, problemSizeGroupConfig, problemSizeG return (resultsFileBaseFinal, benchmarkTestFails) -def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, cCompiler: str, outputPath: Path, buildTmpPath: Path): +def main( + config, + useCache, + asmToolchain: AssemblyToolchain, + srcToolchain: SourceToolchain, + cCompiler: str, + outputPath: Path, + buildTmpPath: Path, + useShortNames: bool, + debugConfig: DebugConfig, + depthUConfig: DepthUConfig, + deviceId: int, + isaInfoMap: Dict[str, IsaInfo] +): """Entry point for the "BenchmarkProblems" section of a Tensile config yaml""" - ClientExecutable.getClientExecutable(srcToolchain.compiler, cCompiler, outputPath) + ClientExecutable.getClientExecutable(str(srcToolchain.compiler.path), cCompiler, outputPath) if config is None: print(f'No config specified in {globalParameters["ConfigPath"]}, built client only') @@ -399,7 +532,7 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source for idx, sizeGroupConfig in enumerate(problemSizeGroupConfigs): print2("ProblemTypeConfig: {}".format(problemTypeConfig)) - problemTypeObj = ProblemType(problemTypeConfig) + problemTypeObj = ProblemType(problemTypeConfig, debugConfig.printIndexAssignmentInfo) # using a suffix to check the csv version (for later addFromCSV()) csvSuffix = "_CSVWinner" if globalParameters["CSVExportWinner"] else "" @@ -418,7 +551,22 @@ def main(config, useCache, asmToolchain: AssemblyToolchain, srcToolchain: Source # benchmark problem size group benchmarkProblemsPath = ensurePath(outputPath / BENCHMARK_PROBLEMS_DIR) (resultsFileBaseFinal, benchmarkErrors) = \ - benchmarkProblemType(problemTypeConfig, sizeGroupConfig, idx, useCache, asmToolchain, srcToolchain, cCompiler, buildTmpPath, benchmarkProblemsPath) + _benchmarkProblemType( + problemTypeConfig, + sizeGroupConfig, + idx, + useCache, + asmToolchain, + srcToolchain, + cCompiler, + buildTmpPath, + benchmarkProblemsPath, + useShortNames, + debugConfig, + depthUConfig, + deviceId, + isaInfoMap + ) totalTestFails += benchmarkErrors print("clientExit={} {} for {}" \ diff --git a/tensilelite/Tensile/BenchmarkStructs.py b/tensilelite/Tensile/BenchmarkStructs.py index 6211083479..8dd27925e5 100644 --- a/tensilelite/Tensile/BenchmarkStructs.py +++ b/tensilelite/Tensile/BenchmarkStructs.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -24,11 +24,15 @@ from copy import deepcopy import itertools -from .Common import print1, print2, hasParam, printExit, \ + +from Tensile.Common.ValidParameters import checkParametersAreValid +from Tensile.Common import print1, print2, hasParam, printExit, \ defaultBenchmarkCommonParameters, validParameters, globalParameters, \ defaultBatchedBenchmarkFinalProblemSizes, defaultBenchmarkFinalProblemSizes +from Tensile.SolutionStructs.Problem import ProblemType + from .CustomKernels import getAllCustomKernelNames -from .SolutionStructs import ProblemType, ProblemSizes, ActivationArgs, BiasTypeArgs, \ +from .SolutionStructs import ProblemSizes, ActivationArgs, BiasTypeArgs, \ FactorDimArgs @@ -43,26 +47,6 @@ def getDefaultsForMissingParameters(paramList, defaultParams): return benchmarkParams -def checkParametersAreValid(param, validParams): - """Ensures paramaters in params exist and have valid values as specified by validParames""" - (name, values) = param - if name == "ProblemSizes": - return - elif name == "InternalSupportParams": - return - - if name not in validParams: - printExit("Invalid parameter name: {}\nValid parameters are {}." \ - .format(name, sorted(validParameters.keys()))) - - for value in values: - if validParams[name] != -1 and value not in validParams[name]: - msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}." - msgExt = " (only first 32 combos printed)\nRefer to Common.py for more info" \ - if len(validParams[name])>32 else "" - printExit(msgBase.format(name, value, name, validParams[name][:32], msgExt)) - - def separateParameters(paramSetList): """Separates paramSetList into parameters with single and multiple values""" singleVaules = {} @@ -92,9 +76,9 @@ def checkCDBufferAndStrides(problemType, problemSizes, isCEqualD): class BenchmarkProcess: """Representation of benchmarking parameters and resulting steps""" - def __init__(self, problemTypeConfig, problemSizeGroupConfig): + def __init__(self, problemTypeConfig, problemSizeGroupConfig, printIndexAssignmentInfo: bool): """Create from the two sections of a config for a BenchmarkProblem""" - self.problemType = ProblemType(problemTypeConfig) + self.problemType = ProblemType(problemTypeConfig, printIndexAssignmentInfo) self.isBatched = "Batched" in problemTypeConfig and problemTypeConfig["Batched"] print2("# BenchmarkProcess beginning {}".format(self.problemType)) diff --git a/tensilelite/Tensile/ClientExecutable.py b/tensilelite/Tensile/ClientExecutable.py index b02a2f0e6a..0d5d399dc1 100644 --- a/tensilelite/Tensile/ClientExecutable.py +++ b/tensilelite/Tensile/ClientExecutable.py @@ -72,9 +72,9 @@ def clientExecutableEnvironment(builddir: Optional[str], cxxCompiler: str, cComp return CMakeEnvironment(sourcedir, builddir, **options) -buildEnv = None +buildEnv = None # why? -def getClientExecutable(cxxCompiler: str, cCompiler: str, builddir): +def getClientExecutable(cxxCompiler: str, cCompiler: str, builddir: Path): if "PrebuiltClient" in globalParameters: return globalParameters["PrebuiltClient"] diff --git a/tensilelite/Tensile/ClientWriter.py b/tensilelite/Tensile/ClientWriter.py index 5d3e31367d..8126b7e043 100644 --- a/tensilelite/Tensile/ClientWriter.py +++ b/tensilelite/Tensile/ClientWriter.py @@ -30,13 +30,17 @@ from pathlib import Path from enum import Enum from glob import glob +from typing import List + +from Tensile.SolutionStructs.Problem import ProblemType, ProblemSizesMock, ProblemSizesMockDummy +from Tensile.SolutionStructs import ActivationArgs, BiasTypeArgs, FactorDimArgs +from Tensile.Toolchain.Component import Assembler from . import ROOT_PATH from . import ClientExecutable from . import LibraryIO -from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock, isaToGfx, \ - LIBRARY_LOGIC_DIR, LIBRARY_CLIENT_DIR -from .SolutionStructs import ProblemType, ProblemSizesMock, ProblemSizesMockDummy, ActivationArgs, BiasTypeArgs, FactorDimArgs +from .Common import globalParameters, ensurePath, print1, printExit, printWarning, ClientExecutionLock, isaToGfx, IsaInfo, \ + LIBRARY_LOGIC_DIR, LIBRARY_CLIENT_DIR, detectGlobalCurrentISA, DepthUConfig from .TensileCreateLibrary import copyStaticFiles from .Contractions import FreeIndex, BatchIndex from .Contractions import ProblemType as ContractionsProblemType @@ -79,7 +83,7 @@ class ClientLogLevel(Enum): ################################################################################ # Main ################################################################################ -def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): +def main(config, assembler: Assembler, cCompiler: str, isaInfoMap, outputPath: Path, deviceId: int, useShortNames: bool=False): libraryLogicPath = ensurePath(outputPath / LIBRARY_LOGIC_DIR) clientLibraryPath = ensurePath(outputPath / LIBRARY_CLIENT_DIR) @@ -97,7 +101,7 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): functions = [] functionNames = [] - createLibraryScript = getBuildClientLibraryScript(clientLibraryPath, libraryLogicPath, cxxCompiler) + createLibraryScript = getBuildClientLibraryScript(clientLibraryPath, libraryLogicPath, str(assembler.path), isaToGfx(list(isaInfoMap.keys())[0]), useShortNames) subprocess.run(shlex.split(createLibraryScript), cwd=clientLibraryPath) coList = glob(os.path.join(clientLibraryPath, "library/*.co")) yamlList = glob(os.path.join(clientLibraryPath, "library/*.yaml")) @@ -105,7 +109,7 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): clientParametersPaths = [] for logicFileName in logicFiles: (scheduleName, _, problemType, _, exactLogic, newLibrary) \ - = LibraryIO.parseLibraryLogicFile(logicFileName, cxxCompiler) + = LibraryIO.parseLibraryLogicFile(logicFileName, assembler, False, False, False, DepthUConfig(), isaInfoMap, globalParameters["LazyLibraryLoading"]) functions.append((scheduleName, problemType)) functionNames.append("tensile_%s" % (problemType)) problemSizes = ProblemSizesMock(exactLogic) if exactLogic else ProblemSizesMockDummy() @@ -137,6 +141,7 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): activationArgs = ActivationArgs(problemType, activationEnums) if isForAll else "" factorDimArgs = FactorDimArgs(problemType, factorDimEnums) + print1(f"libraryFile: {yamlList}") clientParametersPaths.append(writeClientConfig( forBenchmark=False, solutions=None, @@ -145,11 +150,12 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): factorDimArgs=factorDimArgs, activationArgs=activationArgs, icacheFlushArgs=icacheFlushArgs, - stepName=str(ProblemType(problemType)), + stepName=str(ProblemType(problemType, False)), stepBaseDir=str(clientLibraryPath), newLibrary=newLibrary, - configBase="ClientParameters_%s"%str(ProblemType(problemType)), + configBase="ClientParameters_%s"%str(ProblemType(problemType, False)), codeObjectFiles=coList, + deviceId=deviceId, tileAwareSelection=False, libraryFile=yamlList[0])) @@ -167,7 +173,7 @@ def main(config, cxxCompiler: str, cCompiler: str, outputPath: Path): forBenchmark = False enableTileSelection = False - returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler, cCompiler, clientLibraryPath, clientParametersPaths) + returncode = runClient(libraryLogicPath, forBenchmark, enableTileSelection, str(assembler.path), cCompiler, clientLibraryPath, clientParametersPaths) return returncode @@ -200,7 +206,7 @@ def runClient(libraryLogicPath, forBenchmark, enableTileSelection, cxxCompiler: return process.returncode -def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler): +def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler, targetGfx, useShortNames: bool=False): import io runScriptFile = io.StringIO() @@ -209,7 +215,7 @@ def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler): if not globalParameters["LazyLibraryLoading"]: callCreateLibraryCmd += " --no-lazy-library-loading" - if globalParameters["ShortNames"]: + if useShortNames: callCreateLibraryCmd += " --short-file-names" if globalParameters.get("AsmDebug", False): @@ -218,7 +224,7 @@ def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler): if globalParameters["KeepBuildTmp"]: callCreateLibraryCmd += " --keep-build-tmp" - callCreateLibraryCmd += " --architecture=" + globalParameters["Architecture"] + callCreateLibraryCmd += " --architecture=" + targetGfx callCreateLibraryCmd += " --code-object-version=" + globalParameters["CodeObjectVersion"] callCreateLibraryCmd += " --cxx-compiler=" + cxxCompiler callCreateLibraryCmd += " --library-format=" + globalParameters["LibraryFormat"] @@ -231,17 +237,6 @@ def getBuildClientLibraryScript(buildPath, libraryLogicPath, cxxCompiler): return runScriptFile.getvalue() -def writeBuildClientLibraryScript(path, libraryLogicPath, cxxCompiler): - filename = os.path.join(path, \ - "build.%s" % ("bat" if os.name == "nt" else "sh") ) - with open(filename, "w") as file: - file.write("#!/bin/bash\n\n") - file.write("set -ex\n") - file.write(getBuildClientLibraryScript(path, libraryLogicPath, cxxCompiler)) - - if os.name != "nt": - os.chmod(filename, 0o777) - return filename def writeRunScript(path, forBenchmark, enableTileSelection, cxxCompiler: str, cCompiler: str, buildDir, configPaths=None): if configPaths is None: @@ -500,7 +495,7 @@ def pruneModeName(mode): if mode == 5: return 'Prune0X0X' if mode == 6: return 'Prune00XX' -def writeClientConfigIni(forBenchmark, problemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, problemType, sourceDir, codeObjectFiles, resultsFileName, parametersFilePath, libraryFile=None): +def writeClientConfigIni(forBenchmark, problemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, problemType, sourceDir, codeObjectFiles, resultsFileName, parametersFilePath, deviceId: int, libraryFile=None): assert os.path.exists(sourceDir), f"sourceDir={sourceDir} does not exist" @@ -513,7 +508,7 @@ def param(key, value): libraryFile = os.path.join(sourceDir, "library", libraryFilename) param("library-file", libraryFile) - currentGFXName = isaToGfx(globalParameters["CurrentISA"]) + currentGFXName = isaToGfx(detectGlobalCurrentISA(deviceId)) for coFile in codeObjectFiles: if 'gfx' not in coFile or currentGFXName in coFile: param("code-object", os.path.join(sourceDir,coFile)) @@ -573,7 +568,7 @@ def param(key, value): if globalParameters["DataInitValueActivationArgs"]: param('activation-additional-args', ','.join(map(str, globalParameters["DataInitValueActivationArgs"]))) - param("device-idx", globalParameters["Device"]) + param("device-idx", deviceId) param("init-seed", globalParameters["DataInitSeed"]) @@ -656,6 +651,7 @@ def writeClientConfig( newLibrary, codeObjectFiles, tileAwareSelection, + deviceId: int, configBase = "ClientParameters", libraryFile = None ): @@ -677,11 +673,11 @@ def writeClientConfig( resultsFileName = os.path.join(stepBaseDir, "../Data", stepName+".csv") newSolution = next(iter(newLibrary.solutions.values())) - writeClientConfigIni(forBenchmark, problemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, newSolution.problemType, sourceDir, codeObjectFiles, resultsFileName, filename, libraryFile) + writeClientConfigIni(forBenchmark, problemSizes, biasTypeArgs, factorDimArgs, activationArgs, icacheFlushArgs, newSolution.problemType, sourceDir, codeObjectFiles, resultsFileName, filename, deviceId, libraryFile) return filename -def CreateBenchmarkClientParametersForSizes(libraryRootPath, problemSizes, dataFilePath, configFile, problemTypeDict=None): +def CreateBenchmarkClientParametersForSizes(libraryRootPath, problemSizes, dataFilePath, configFile, deviceId, problemTypeDict=None): libraryPath = os.path.join(libraryRootPath, "library") libraryFiles = [os.path.join(libraryPath, f) for f in os.listdir(libraryPath)] @@ -698,4 +694,4 @@ def CreateBenchmarkClientParametersForSizes(libraryRootPath, problemSizes, dataF problemTypeDict = metaData["ProblemType"] problemType = ContractionsProblemType.FromOriginalState(problemTypeDict) - writeClientConfigIni(True, problemSizes, "", "", "", "", problemType, libraryRootPath, codeObjectFiles, dataFilePath, configFile) + writeClientConfigIni(True, problemSizes, "", "", "", "", problemType, libraryRootPath, codeObjectFiles, dataFilePath, configFile, deviceId) diff --git a/tensilelite/Tensile/Common/Architectures.py b/tensilelite/Tensile/Common/Architectures.py index d412d1e1fd..e13ed074c3 100644 --- a/tensilelite/Tensile/Common/Architectures.py +++ b/tensilelite/Tensile/Common/Architectures.py @@ -21,9 +21,11 @@ ################################################################################ import re -from typing import Optional +from subprocess import run, PIPE +from typing import List, Optional from .Types import IsaVersion +from .Utilities import locateExe # Translate GPU targets to filter filenames in Tensile_LOGIC directory architectureMap = { @@ -54,6 +56,50 @@ "gfx1201": "gfx1201", } +gfxVariantMap = { + "gfx906": ["gfx906:xnack+", "gfx906:xnack-"], + "gfx908": ["gfx908:xnack+", "gfx908:xnack-"], + "gfx90a": ["gfx90a:xnack+", "gfx90a:xnack-"], + "gfx942": ["gfx942:xnack+", "gfx942:xnack-"], +} + +SUPPORTED_ISA = [ + IsaVersion(8, 0, 3), + IsaVersion(9, 0, 0), + IsaVersion(9, 0, 6), + IsaVersion(9, 0, 8), + IsaVersion(9, 0, 10), + IsaVersion(9, 4, 0), + IsaVersion(9, 4, 1), + IsaVersion(9, 4, 2), + IsaVersion(10, 1, 0), + IsaVersion(10, 1, 1), + IsaVersion(10, 1, 2), + IsaVersion(10, 3, 0), + IsaVersion(11, 0, 0), + IsaVersion(11, 0, 1), + IsaVersion(11, 0, 2), + IsaVersion(12, 0, 0), + IsaVersion(12, 0, 1), +] + + +def isaToGfx(arch: IsaVersion) -> str: + """Converts an ISA version to a gfx architecture name. + + Args: + arch: An object representing the major, minor, and step version of the ISA. + + Returns: + The name of the GPU architecture (e.g., 'gfx906'). + """ + # Convert last digit to hex because reasons + name = str(arch[0]) + str(arch[1]) + ("%x" % arch[2]) + return "gfx" + "".join(map(str, name)) + + +SUPPORTED_GFX = [isaToGfx(isa) for isa in SUPPORTED_ISA] + def gfxToIsa(name: str) -> Optional[IsaVersion]: """Extracts the ISA version from a given gfx architecture name. @@ -76,21 +122,7 @@ def gfxToIsa(name: str) -> Optional[IsaVersion]: ipart = ipart[:-1] major = int(ipart) - return tuple((major, minor, step)) - - -def isaToGfx(arch: IsaVersion) -> str: - """Converts an ISA version to a gfx architecture name. - - Args: - arch: An object representing the major, minor, and step version of the ISA. - - Returns: - The name of the GPU architecture (e.g., 'gfx906'). - """ - # Convert last digit to hex because reasons - name = str(arch[0]) + str(arch[1]) + ("%x" % arch[2]) - return "gfx" + "".join(map(str, name)) + return IsaVersion(major, minor, step) def gfxToSwCodename(gfxName: str) -> Optional[str]: @@ -110,3 +142,67 @@ def gfxToSwCodename(gfxName: str) -> Optional[str]: if gfxName in archKey: return architectureMap[archKey] return None + + +def gfxToVariants(gfx: str) -> List[str]: + """Retrieves the list of variants for a given gfx architecture name. + + Args: + gfx: The name of the GPU architecture (e.g., 'gfx906'). + + Returns: + List of variants for the GPU architecture. + """ + return gfxVariantMap.get(gfx, [gfx]) + + +def cliArchsToIsa(cliArchs: str) -> List[IsaVersion]: + """Maps the requested gfx architectures to ISA numbers. + + Args: + archs: str of ";" or "_" separated gfx architectures (e.g., gfx1100 or gfx90a;gfx1101). + + Returns: + List of tuples + """ + archs = cliArchs.split(";") if ";" in cliArchs else cliArchs.split("_") + return SUPPORTED_ISA if "all" in archs else [gfxToIsa(''.join(map(str, arch))) for arch in archs] + + +def _detectGlobalCurrentISA(detectionTool, deviceId: int): + """ + Returns returncode if detection failure + """ + process = run([detectionTool], stdout=PIPE) + archList = [] + for line in process.stdout.decode().split("\n"): + arch = gfxToIsa(line.strip()) + if arch is not None: + if arch in SUPPORTED_ISA: + print(f"# Detected GPU {deviceId} with ISA: " + isaToGfx(arch)) + archList.append(arch) + if process.returncode: + print(f"{detectionTool} exited with code {process.returncode}") + return archList[deviceId] if (len(archList) > 0 and process.returncode == 0) else process.returncode + + +# locateExe silently fails which is not good +AMDGPUArchPath = locateExe( + "/opt/rocm", "llvm/bin/amdgpu-arch" +) + +ROCmAgentEnumeratorPath = locateExe( + "/opt/rocm/bin","rocm_agent_enumerator" +) + +def detectGlobalCurrentISA(deviceId: int): + """ + Returns returncode if detection failure + """ + result = _detectGlobalCurrentISA(AMDGPUArchPath, deviceId) + if not isinstance(result, IsaVersion): + print("Attempting to detect ISA with rocm_agent_enumerator") + result = _detectGlobalCurrentISA(ROCmAgentEnumeratorPath, deviceId) + if not isinstance(result, IsaVersion): + raise Exception("Failed to detect currect ISA") + return result diff --git a/tensilelite/Tensile/Common/Capabilities.py b/tensilelite/Tensile/Common/Capabilities.py index db20079d5a..00dcde5458 100644 --- a/tensilelite/Tensile/Common/Capabilities.py +++ b/tensilelite/Tensile/Common/Capabilities.py @@ -1,12 +1,37 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + import subprocess from functools import lru_cache -from typing import Tuple +from typing import List, Dict from .Architectures import isaToGfx +from .Types import IsaVersion, IsaInfo def _tryAssembler( - isaVersion: Tuple[int, int, int], + isaVersion: IsaVersion, assemblerPath: str, asmString: str, debug: bool = False, @@ -164,7 +189,7 @@ def initArchCaps(isaVersion) -> dict: rv["SDWAWait"] = (isaVersion in [(9,4,0), (9,4,1), (9,4,2)]) rv["VgprBank"] = (isaVersion[0] in (10, 11, 12)) rv["DSLow16NotPreserve"] = isaVersion[0] == (12) - rv["WrokGroupIdFromTTM"] = isaVersion[0] == (12) + rv["WorkGroupIdFromTTM"] = isaVersion[0] == (12) rv["NoSDWA"] = isaVersion[0] == (12) rv["VOP3ByteSel"] = isaVersion[0] == (12) rv["HasFP8_OCP"] = isaVersion[0] == (12) @@ -211,3 +236,13 @@ def initAsmBugs(asmCaps) -> dict: rv["ExplicitNC"] = asmCaps["HasExplicitNC"] return rv + +def makeIsaInfoMap(targetIsas: List[IsaVersion], cxxCompiler: str) -> Dict[IsaVersion, IsaInfo]: + isaInfoMap = {} + for v in targetIsas: + asmCaps = initAsmCaps(v, cxxCompiler, False) + archCaps = initArchCaps(v) + regCaps = initRegisterCaps(v, archCaps) + asmBugs = initAsmBugs(asmCaps) + isaInfoMap[v] = IsaInfo(asmCaps, archCaps, regCaps, asmBugs) + return isaInfoMap diff --git a/tensilelite/Tensile/Common/GlobalParameters.py b/tensilelite/Tensile/Common/GlobalParameters.py index 9c6bdf6147..0ebf9a4227 100644 --- a/tensilelite/Tensile/Common/GlobalParameters.py +++ b/tensilelite/Tensile/Common/GlobalParameters.py @@ -22,6 +22,7 @@ # ################################################################################ +import itertools import math import os.path import subprocess @@ -29,12 +30,15 @@ import time from collections import OrderedDict from copy import deepcopy +from typing import Dict from Tensile import __version__ -from .Architectures import gfxToIsa, isaToGfx -from .Capabilities import initArchCaps, initAsmBugs, initAsmCaps -from .Utilities import locateExe, versionIsCompatible +from .Architectures import isaToGfx, SUPPORTED_ISA +from .Types import IsaVersion, IsaInfo +from .Utilities import locateExe, versionIsCompatible, print1, print2, printExit, printWarning, \ + verbosity +from .ValidParameters import validParameters startTime = time.time() @@ -45,9 +49,6 @@ globalParameters["PerformanceMetric"] = ( "DeviceEfficiency" # performance metric for benchmarking; one of {DeviceEfficiency, CUEfficiency} ) -globalParameters["PrintLevel"] = ( - 1 # how much info to print in generator. 0=none, 1=standard, 2=verbose -) globalParameters["ClientLogLevel"] = ( 3 # the log level of client. 0=Error, 1=Terse, 2=Verbose, 3=Debug (Aligned with ResultReporter.hpp) ) @@ -57,9 +58,9 @@ True # T=On hip, use the timestamps for kernel start and stop rather than separate events. Can provide more accurate kernel timing. For GlobalSplitU kernels, recommend disabling this to provide consistent ) # timing between GSU / non-GSU kernels -globalParameters["CodeFromFiles"] = ( - True # if False byte arrays will be generated during Benchmarking phase as before -) +#globalParameters["CodeFromFiles"] = ( +# True # if False byte arrays will be generated during Benchmarking phase as before +#) globalParameters["PinClocks"] = False # T=pin gpu clocks and fan, F=don't globalParameters["HardwareMonitor"] = ( True # False: disable benchmarking client monitoring clocks using rocm-smi. @@ -130,15 +131,7 @@ # even if error occurs in kernel generation (ie due to resource overflow), # generate the kernel source anyway. Tensile will also attempt to run # the kernel. Useful to examine and debug overflow errors. -globalParameters["ForceGenerateKernel"] = 0 - -######################################## -# optimization knob controls -######################################## - -globalParameters["UnrollLoopEfficiencyEnable"] = ( - False # if True split(S) MAC&LDS in each unroll iteration into n smaller groups.. -) +# globalParameters["ForceGenerateKernel"] = 0 ######################################## # less common @@ -146,9 +139,9 @@ globalParameters["CMakeBuildType"] = ( "Release" # whether benchmark clients and library client should be release or debug ) -globalParameters["PrintSolutionRejectionReason"] = ( - False # when a solution is marked as invalid, print why -) +#globalParameters["PrintSolutionRejectionReason"] = ( +# False # when a solution is marked as invalid, print why +#) globalParameters["LogicFormat"] = "yaml" # set library backend (yaml, or json) globalParameters["LibraryFormat"] = "yaml" # set library backend (yaml, or msgpack) @@ -205,23 +198,15 @@ # build parameters globalParameters["CMakeCXXFlags"] = "" # pass flags to cmake globalParameters["CMakeCFlags"] = "" # pass flags to cmake -globalParameters["DebugKernel"] = ( - False # assembly only, kernel gets buffer for debug "printing"; kernel writes data to memory, gets coppied to host and printed -) +#globalParameters["DebugKernel"] = ( +# False # assembly only, kernel gets buffer for debug "printing"; kernel writes data to memory, gets coppied to host and printed +#) globalParameters["AsanBuild"] = False # build with asan -globalParameters["SaveTemps"] = False # Generate intermediate results of hip kernels +#globalParameters["SaveTemps"] = False # Generate intermediate results of hip kernels globalParameters["KeepBuildTmp"] = False # If true, do not remove artifacts in build_tmp # debug for assembly -globalParameters["EnableAsserts"] = False # Enable assembly debug assert -globalParameters["EnableDebugA"] = False # Enable / Disable CheckValue1A -globalParameters["EnableDebugB"] = False # Enable / Disable CheckValue1B -globalParameters["EnableDebugC"] = False # Enable / Disable CheckValueC -globalParameters["ExpectedValueC"] = 16.0 # Expected C Value when CheckValueC, debug for Alpha*A*B -globalParameters["ForceCExpectedValue"] = ( - False # Force C to "DebugExpectedValueC", debug for global write -) -globalParameters["SplitGSU"] = False # Split GSU kernel into GSU1 and GSUM +#globalParameters["SplitGSU"] = False # Split GSU kernel into GSU1 and GSUM # Tensor printing controls: globalParameters["PrintTensorA"] = 0 # Print TensorA after initialization @@ -237,7 +222,7 @@ ) globalParameters["PrintTensorBias"] = 0 # Print TensorBias after initialization globalParameters["PrintTensorAmaxD"] = 0 # Print AmaxD after validation -globalParameters["PrintIndexAssignments"] = 0 # Print the tensor index assignment info +#globalParameters["PrintIndexAssignments"] = 0 # Print the tensor index assignment info globalParameters["PrintWinnersOnly"] = False # Only print the solutions which become the fastest globalParameters["PrintCodeCommands"] = ( False # print the commands used to generate the code objects (asm,link,hip-clang, etc) @@ -251,34 +236,8 @@ # device selection globalParameters["Platform"] = 0 # select opencl platform -globalParameters["Device"] = 0 # select hip device or opencl device within platform # shouldn't need to change -globalParameters["DeviceLDS"] = 65536 # LDS bytes per CU, for computing occupancy -globalParameters["MaxLDS"] = 65536 # max LDS a kernel should attempt to use -globalParameters["ShortNames"] = ( - False # on windows kernel names can get too long; =True will convert solution/kernel names to serial ids -) - -globalParameters["SupportedISA"] = [ - (8, 0, 3), - (9, 0, 0), - (9, 0, 6), - (9, 0, 8), - (9, 0, 10), - (9, 4, 2), - (10, 1, 0), - (10, 1, 1), - (10, 1, 2), - (10, 3, 0), - (11, 0, 0), - (11, 0, 1), - (11, 0, 2), - (12, 0, 0), - (12, 0, 1), -] # assembly kernels writer supports these architectures - -globalParameters["NewClient"] = 2 # Old client deprecated: NewClient must be set to 2. globalParameters["ClientExecutionLockPath"] = ( None # Path for a file lock to ensure only one client is executed at once. filelock module is required if this is enabled. ) @@ -290,7 +249,6 @@ ) # internal, i.e., gets set during startup -globalParameters["CurrentISA"] = (0, 0, 0) globalParameters["AMDGPUArchPath"] = None # /opt/rocm/llvm/bin/amdgpu-arch globalParameters["ROCmAgentEnumeratorPath"] = None # /opt/rocm/bin/rocm_agent_enumerator globalParameters["ROCmSMIPath"] = None # /opt/rocm/bin/rocm-smi @@ -298,12 +256,11 @@ # default runtime is selected based on operating system, user can override if os.name == "nt": - globalParameters["RuntimeLanguage"] = "HIP" # "OCL" + globalParameters["RuntimeLanguage"] = "HIP" else: globalParameters["RuntimeLanguage"] = "HIP" globalParameters["CodeObjectVersion"] = "4" -globalParameters["Architecture"] = "all" # perf model globalParameters["PerfModelL2ReadHits"] = 0.0 @@ -313,7 +270,7 @@ # limitation for training globalParameters["MaxWorkspaceSize"] = 128 * 1024 * 1024 # max workspace for training (128MB) -globalParameters["MinKForGSU"] = 32 # min K size to use GlobalSplitU algorithm (only for HPA now) +#globalParameters["MinKForGSU"] = 32 # min K size to use GlobalSplitU algorithm (only for HPA now) # control if a solution is run for a given problem globalParameters["GranularityThreshold"] = 0.0 @@ -348,6 +305,7 @@ globalParameters["UseEffLike"] = True # Set to False to use winnerGFlops as the performance metric # Save a copy - since pytest doesn't re-run this initialization code and YAML files can override global settings - odd things can happen +# we should do this here... defaultGlobalParameters = deepcopy(globalParameters) @@ -375,889 +333,6 @@ "UseUniversalArgs": True, } - -################################################################################ -# Enumerate Valid Solution Parameters -################################################################################ -validWorkGroups = [] -for numThreads in range(32, 1025, 32): - for nsg in [1, 2, 4, 8, 16, 32, 64, 96, 128, 256]: - for sg0 in range(1, numThreads // nsg + 1): - sg1 = numThreads // nsg // sg0 - if sg0 * sg1 * nsg == numThreads: - workGroup = [sg0, sg1, nsg] - validWorkGroups.append(workGroup) - -validThreadTileSides = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + list( - range(20, 256, 4) -) -validThreadTiles = [] -for i in validThreadTileSides: - for j in validThreadTileSides: - validThreadTiles.append([i, j]) - -validActivationFormats = ("NCHW", "NHWC", "CNHW", "NCDHW", "NDHWC", "CNDHW") -validWeightFormats = ("KCYX", "KYXC", "CKYX", "CYXK", "KCZYX", "CKZYX", "CZYXK") -validMacroTileSides = [ - 1, - 2, - 4, - 8, - 16, - 32, - 64, - 128, - 256, - 512, - 1024, - 6, - 12, - 24, - 48, - 96, - 192, - 384, - 768, -] -validMacroTiles = [] -validISA = [(0, 0, 0)] -validISA.extend(globalParameters["SupportedISA"]) -depthUs = list(range(2, 1024 + 1, 1)) -for i in validMacroTileSides: - for j in validMacroTileSides: - validMacroTiles.append([i, j]) - -validMFMA = {} -validMFMA["H"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] -validMFMA["S"] = [[32, 32, 1, 2], [32, 32, 2, 1], [16, 16, 1, 4], [16, 16, 4, 1], [4, 4, 1, 16]] -validMFMA["B"] = [[32, 32, 2, 2], [32, 32, 4, 1], [16, 16, 2, 4], [16, 16, 8, 1], [4, 4, 2, 16]] -validMFMA["4xi8"] = [ - [32, 32, 4, 2], - [32, 32, 8, 1], - [16, 16, 4, 4], - [16, 16, 16, 1], - [4, 4, 4, 16], - [32, 32, 16, 1], - [16, 16, 32, 1], -] -validMFMA["D"] = [[16, 16, 4, 1], [4, 4, 4, 4]] -validMFMA["B1k"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] -validMFMA["C"] = validMFMA["S"] -validMFMA["Z"] = validMFMA["D"] -validMFMA["I8"] = [ - [32, 32, 4, 2], - [32, 32, 8, 1], - [16, 16, 4, 4], - [16, 16, 16, 1], - [4, 4, 4, 16], -] + [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1]] -validMFMA["F8"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["B8"] = validMFMA["F8"] -validMFMA["F8B8"] = validMFMA["F8"] -validMFMA["B8F8"] = validMFMA["F8"] -validMFMA["F8N"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["B8N"] = validMFMA["F8N"] -validMFMA["F8B8N"] = validMFMA["F8N"] -validMFMA["B8F8N"] = validMFMA["F8N"] -validWMMA = [ - [16, 16, 16, 1], -] -validTT = 32 -validMFMA["_format9"] = [] - -for MFMA in [ - validMFMA["H"], - validMFMA["S"], - validMFMA["B"], - validMFMA["D"], - validMFMA["X"], - validMFMA["F8N"], - validWMMA, -]: - for MI in MFMA: - for bm in range(int(math.log(MI[3], 2)) + 1): - for tt0 in range(1, validTT + 1): - for tt1 in range(1, validTT + 1): - for wave_m in range(3): - for wave_n in range(3): - validMFMA["_format9"].append( - [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] - ) -validMatrixInstructions = ( - [[], [-1]] - + validMFMA["H"] - + validMFMA["S"] - + validMFMA["B"] - + validMFMA["D"] - + validMFMA["B1k"] - + validMFMA["X"] -) -validMatrixInstructions = validMatrixInstructions + validMFMA["_format9"] - -validSMFMA = {} -validSMFMA["H"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validSMFMA["B"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validSMFMA["4xi8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["I8"] = validSMFMA["4xi8"] -validSMFMA["F8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["B8"] = validSMFMA["F8"] -validSMFMA["F8B8"] = validSMFMA["F8"] -validSMFMA["B8F8"] = validSMFMA["F8"] -validSMFMA["F8N"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["B8N"] = validSMFMA["F8N"] -validSMFMA["F8B8N"] = validSMFMA["F8N"] -validSMFMA["B8F8N"] = validSMFMA["F8N"] -validSMFMA["_format9"] = [] -for SMFMA in [validSMFMA["H"], validSMFMA["B"], validSMFMA["4xi8"], validSMFMA["F8N"]]: - for MI in SMFMA: - for bm in range(int(math.log(MI[3], 2)) + 1): - for tt0 in range(1, validTT + 1): - for tt1 in range(1, validTT + 1): - for wave_m in range(3): - for wave_n in range(3): - validSMFMA["_format9"].append( - [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] - ) -validSparseMatrixInstructions = validSMFMA["H"] + validSMFMA["B"] + validSMFMA["4xi8"] -validMatrixInstructions = ( - validMatrixInstructions + validSparseMatrixInstructions + validSMFMA["_format9"] -) - - -# The supported typed GEMM, each entry is (Ti, To, Tc). -# DataType (Ti) = The data-type of the input matrices: A/B -# DestDataType (To) = The data-type of the output matrices: C/D -# ComputeDataType (Tc) = The data-type of computation: alpha/beta: -# Cinternal: basically should == ComputeDataType - -# This is used in SolutionStruct.py::checkIfSupportedGEMMType() -validGEMMTypes = [ - ("H", "H", "H"), - ("S", "S", "S"), - ("D", "D", "D"), - ("C", "C", "C"), - ("Z", "Z", "Z"), - ("H", "H", "S"), - ("H", "S", "S"), - ("B", "B", "S"), - ("B", "S", "S"), - ("B", "H", "S"), - ("I8", "I", "I"), - ("4xi8", "I", "I"), - ("I8", "I8", "I"), - ("I8", "I", "S"), - ("I8", "I8", "S"), - ("I8", "H", "S"), - ("I8", "B", "S"), - ("F8", "S", "S"), - ("B8", "S", "S"), - ("F8B8", "S", "S"), - ("B8F8", "S", "S"), - ("F8", "H", "S"), - ("B8", "H", "S"), - ("F8B8", "H", "S"), - ("B8F8", "H", "S"), - ("B8", "B", "S"), - ("H", "F8", "S"), - ("F8", "B", "S"), - ("F8B8", "B", "S"), - ("B8F8", "B", "S"), # in/out are both R8 - ("F8", "F8", "S"), - ("B8", "B8", "S"), - ("F8B8", "B8", "S"), - ("B8F8", "B8", "S"), - ("F8", "B8", "S"), - ("B8", "F8", "S"), - ("F8B8", "F8", "S"), - ("B8F8", "F8", "S"), # F8 NANOO - ("F8N", "S", "S"), - ("B8N", "S", "S"), - ("F8B8N", "S", "S"), - ("B8F8N", "S", "S"), - ("F8N", "H", "S"), - ("B8N", "H", "S"), - ("F8B8N", "H", "S"), - ("B8F8N", "H", "S"), - ("B8N", "B", "S"), - ("H", "F8N", "S"), - ("F8N", "B", "S"), - ("F8B8N", "B", "S"), - ("B8F8N", "B", "S"), # in/out are both R8 - ("F8N", "F8N", "S"), - ("B8N", "B8N", "S"), - ("F8B8N", "B8N", "S"), - ("B8F8N", "B8N", "S"), - ("F8N", "B8N", "S"), - ("B8N", "F8N", "S"), - ("F8B8N", "F8N", "S"), - ("B8F8N", "F8N", "S"), -] - -# All HPA types are listed here (HPA=T). The name of the library logic files for these types is: -# *_TiToTc_BH*.yaml where Ti, To, and Tc are the data types of A/B, C/D, and computation, respectively. -# The name of the library logic files for non-HPA (HPA=F) types is: *_TiB*.yaml. -HPATypes = [ - ("H", "S", "S"), - ("H", "H", "S"), - ("B", "B", "S"), - ("B", "S", "S"), - ("B", "H", "S"), - ("I8", "I", "I"), - ("4xi8", "I", "I"), - ("I8", "I", "S"), - ("I8", "I8", "S"), - ("I8", "H", "S"), - ("I8", "B", "S"), - ("F8", "S", "S"), - ("B8", "S", "S"), - ("F8B8", "S", "S"), - ("B8F8", "S", "S"), - ("F8", "H", "S"), - ("B8", "H", "S"), - ("F8B8", "H", "S"), - ("B8F8", "H", "S"), - ("H", "F8", "S"), - ("F8", "B", "S"), - ("F8B8", "B", "S"), # in/out are both R8 - ("F8", "F8", "S"), - ("B8", "B8", "S"), - ("F8B8", "B8", "S"), - ("B8F8", "B8", "S"), - ("F8", "B8", "S"), - ("B8", "F8", "S"), - ("F8B8", "F8", "S"), - ("B8F8", "F8", "S"), - ("F8N", "S", "S"), - ("B8N", "S", "S"), - ("F8B8N", "S", "S"), - ("B8F8N", "S", "S"), - ("F8N", "H", "S"), - ("B8N", "H", "S"), - ("F8B8N", "H", "S"), - ("B8F8N", "H", "S"), - ("H", "F8N", "S"), - ("F8N", "B", "S"), - ("F8B8N", "B", "S"), # in/out are both R8 - ("F8N", "F8N", "S"), - ("B8N", "B8N", "S"), - ("F8B8N", "B8N", "S"), - ("B8F8N", "B8N", "S"), - ("F8N", "B8N", "S"), - ("B8N", "F8N", "S"), - ("F8B8N", "F8N", "S"), - ("B8F8N", "F8N", "S"), -] - -validParameters = { - # 0: Global read is along parallel direction in thread level, - # each load instruction stride whole threads. - # ----> perp - # | [w0, w0, w1,w1,w2,w2,w3,w3, w0, w0, w1,w1,w2,w2,w3,w3] - # | [ t0,t32] [ ] [ t0,t32] [ ] - # para | [ t1,t33] [ wave 1,2,3 ] [ t1,t33] [ wave 1,2,3 ] - # | [ .., ..] [ ] [ .., ..] [ ] - # | [t31,t63] [ ] [t31,t63] [ ] - # V [-load_1] [-load_2] - # - # 1: Each wave load a block of memory, - # each load instruction stride 64 threads. - # ----> perp - # [ w0, w0, w0, w0, w1,w1,w1,w1, w2,w2,w2,w2, w3,w3,w3,w3] - # | [ t0,t32][ t0,t32] - # para | [ t1,t33][ t1,t33] - # | [ .., ..][ .., ..] - # | [t31,t63][t31,t63] - # V [-load_1][-load_2] - # - # - # 2: Each load instruction spread threads evenly in the perp direction - # ----> perp - # | [w0, w1, w2, w3, w0, w1, w2, w3, w0, w1, w2, w3, w0, w1, w2, w3] - # | [t0 ] [t0 ] [t32] [t32] - # para | [t1 ] [t1 ] [t33] [t33] - # | [.. ] [.. ] [.. ] [.. ] - # | [t31] [t31] [t63] [t63] - # V [load_1] [load_2] [load_1] [load_2] - # - "WaveSeparateGlobalReadA": [0, 1, 2], - "WaveSeparateGlobalReadB": [0, 1, 2], - # Add an unrolled loop and NGLL loop with swapped GRA and GRB order. - # which may change the tlb thrashing behavior. - "UnrollLoopSwapGlobalReadOrder": [0, 1], - # PrefetchGlobalRead = 1: - # Requires 2X LDS space, and VGPRs for buffering data on way into LDS - # prefetch / double-buffer reads from global memory -> vgprs -> lds. - # - # PrefetchGlobalRead = 2: - # Do another prefetch while writing data from vgpr to lds. - # prefetch / double-buffer reads from global memory -> vgprs --> lds. - # |-> prefetch reads - "PrefetchGlobalRead": [0, 1, 2], - # number of iteration prefetch local reads from lds to VGPRs buffer = PLR - "PrefetchLocalRead": list(range(128 + 1)), - # MatrixInstruction Only - # If set ClusterLocalRead, each iteration dedicated vgprBuffer for localRead - # So we can schedule these localReads to the front of the loop - "ClusterLocalRead": [0, 1], - # We use double LDS buffer when PrefetchGlobalRead. - # While it reads data from LDS[0]/[1], it prefetch global data and writes to LDS[1]/[0] - # If we can make sure all data are read from LDS to register before writing data to LDS, we can use 1 LDS buffer to save LDS memory. - # this can help to generate Kernel that LDS usage originally exceed MaxLDS if using double LDS buffer, - # or help to increase Occupancy. - # 1 means: Force to use 1 LDS Buffer even with PrefetchGlobalRead - # -1 means: generator will use 1 LDS buffer only when LDS exceed MaxLDS - # Use case: - # SIA2: 1LDSBuffer is set to 1 natively - # SIA3: 1LDSBuffer works only when PGR=True - # TODO: optimize scheduling to support more cases. - "1LDSBuffer": [-1, 0, 1], - # Split the unroll summation into multiple sections and combine the sections - # GSU applies only to the unroll summation dimension - # Set to 0 to disable GSU, kernel code will be generated without GSU support - "GlobalSplitU": list(range(0, 1024 + 1)), - # choose how to do GlobalSplitU - # 1: use atomic operation to accumulate on one buffer - # 2: each GSU group write to each own buffer and accumulate by another kernel - # 3: each GSU group write to each own buffer and accumulate by same kernel - "GlobalSplitUAlgorithm": ["SingleBuffer", "MultipleBuffer", "MultipleBufferSingleKernel"], - # don't create a whole copy of the Unroll loop with loads removed - instead - # use buffer limits to suppress global loads and ignore unnecessary ds_reads - "SuppressNoLoadLoop": [False, True], - # For PrefetchGlobalRead=1, create a second copy of the unroll loop with - # the LDS pointer swaps expanded into inline constants for LDS read and write instructions - # This eliminates 4 vector XOR instructions used for pointer swap - "ExpandPointerSwap": [False, True], - # Schedule global reads and global read increments into LocalRead iterations - # Can reduce pressure on local read instruction dispatch queue - # 0=perform global reads at start of instruction loop - # 1=schedule into the local read instruction iterations - "ScheduleGlobalRead": [0, 1], - # Schedule local writes into LocalRead iterations. - # Can reduce pressure on local read instruction dispatch queue - "ScheduleLocalWrite": [0, 1], - # Scheduling algorithm to use for each iteration: - # 0 = minimal/no scheduling. Global Read and increments, followed by local reads, - # followed by local writes, followed by MACs - "ScheduleIterAlg": [0, 1, 2, 3], - # For MatrixInstruction and SIA3, number of GlobalReadInstruction between mfma - # the purpose of this parameter is to control density of global read instruction scheduling - # Scheduling global read back to back can have better memory efficiency - # However, when full of vmem FIFO, it will block other instruction to be issued - # Range from 0.01 to 32 - # 0.1 means 1 GR per 10 mfma - # 5 means 5 GR per 1 mfma - "GlobalReadPerMfma": [i / 100 for i in range(1, 3200)], - # - # For MatrixInstruction and SIA3, number of LocalWriteInstruction between mfma - # the purpose of this parameter is to control density of local write instruction scheduling - # In PGR1, we want to schedule local write more denser, so we can have more - # latency to hide global read - # In PGR2, since LW is followed by GR, every LW has same whole loop latency - # to hide global read. We want to schedule LW less denser, can - # avoid full of vmem FIFO. - # Range from 0.01 to 32 - # 0.1 means 1 LW per 10 mfma - # 5 means 5 LW per 1 mfma - # -1 will derived an optimized value internally - # -2 will derived an optimized value and override LWPM silently (debug only, not recommended) - "LocalWritePerMfma": [i / 100 for i in range(1, 3200)] + [-1], - # Interleave alpha scale calculation with beta loads and address calcs - rather - # than as a separate block of instructions - "InterleaveAlpha": [0, 1], - # Create a copy of NoLoadLoop which interleaves the stores with the final mac - # calculation and may perform other optimizations - # 0 = no interleave - # 1 = interleave one stores after required macs have completed execution - # 2 = interleave two stores after required macs have completed execution - "OptNoLoadLoop": [0, 1, 2], - "BufferLoad": [False, True], - "BufferStore": [False, True], - # Attempt to load directly from global memory into Vgpr. - # Assembly only - "DirectToVgprA": [False, True], - "DirectToVgprB": [False, True], - "DirectToVgprSparseMetadata": [False, True], - # Attempt to load directly from global memory into LDS. - # Assembly only - # Requires BufferLoad, assembler support for lds modifier on buffer - # loads (checked automatically), GlobalVectorWidth=1 (this is hw - # requirement) and A/B must not require any transpose. - # DirectToLds reduces load latency and eliminates the - # G2L registers used to stage data. Also replaces the - # local write offset with an SGPR. - # For an 8x8 TT with PrefetchGlobalRead=1 this can save 33 VGPRs. - # - Requirements for DirectToLds=1: - # GlobalReadVectorWidth = 1/2/4 (GRVW * bpe must be 4 for now) - # TransposeLDS = 1 for TLU=0 case - # DirectToLds support for x1 only for now - "DirectToLds": [False, True], - # Load options: - # (GRO = Global Read Offset) - # BufferLoad=0: - # = Use flat instructions with 64 bit GRO for each load - # + supports sizes up to 2^64 - # - uses many VGPR for addressing - # - uses execmask+compares for edge detection - # - generates extra LDS traffic (could convert flat->global load) - # BufferLoad=1: - # = Use buffer load instructions with 32-bit offset - # + Less VGPRS (32b offset vs 64-bit) needed for addressing - # + Uses hardware buffer limit for edge detection - # - Limited range - the bot-right corner of macro-tile (plus padding=GRVW - # for shift-pointer, if ShiftPtr is required) must be within 2^32. - # ShiftPtrPad = MayShift ? GRWV*BPE : 0 - # For TLU=1: Unroll*StrideA1 + ShiftPtrPad <= 2^32 - # For TLU=0: MT*StrideA1 + ShiftPtrPad <= 2^32 - # These conditions should be checked using Assert - TODO - # = UseSgprForGRO=1: - # + Attempt to use SGPR for Global Read Offsets. - # + Use one VGPR base GRO + many SGPR GRO rather than many VGPR GRO. - # + Each SGPR stores an offset from base GlobalReadOffset+0. - # - Requirements for UseSgprForGRO=1: - # - BufferLoad=1 - # - Use appropriate Assert*ElementMultiple or GRVW=1 to eliminate need for ShifPtr - # (UseSgprForGRO does not support ShiftPtr since ShiftPtr needs to potentially shift GRO) - # = KernelWriterAssembly also supports 64-bit 2D buffer size (see use64bPbcLimit) - # - Requires 4 instructions to move scalar limit and a couple SGPR - # - Enabled by default. If the overhead matters we can add asserts/YAML parm to specialize - # = UseInstOffsetForGRO=1: - # + Attempt to use Instruction offset for Global Read Offsets. - # + This feature avoid updating m0 for subsequent GRO(s) for directToLds feature - # - Requirements for UseInstOffsetForGRO=1: - # - BufferLoad=1 - # - DirectToLds=1 - # converting m0 update from LocalWriteAddrSGpr using is usually win - # -1 attempt to use a heuristic to determine when the tile size will use too many SGPR and fall back to VGPR - "UseInstOffsetForGRO": [-1, 0, 1], - # Converting VGPR GRO into SGPR GRO is usually a win - # However, the mode may exhaust all available SGPR, in particular for large unroll - # -1 attempt to use a heuristic to determine when the tile size will use too many SGPR and fall back to VGPR - "UseSgprForGRO": [-1, 0, 1], - # Use a 64-bit shadow limit register to allow buffers larger than 2^32 bytes - "Use64bShadowLimit": [True, False], - # Assertion properties - # These provide information or assertions that the problem size meets certain requirements - # for sizes or alignments. The kernel generator can use this information to produce - # a kernel which uses those assertions to produce a faster kernel. - # - # If modifying or adding Assertions also change ProblemProperties class in TensileTypes.h - # Kernel generator will assume that the summation size is some multiple of the element size - # and uses this to optimize the kernel. - # This can result in more efficient kernels, but requires runtime checking to ensure the specified - # summation value meets the requirements. - # (Recommended AF1EM value is 8 for half, 4 for single, 2 for double) - # - # Optimizations enabled by AssertSummationElementMultiple>1: - # - If >=2 for half: - # - Tail loop loads can be vectorized 2X to use dword - # - Enables asm kernels on V20 - # - Can use DirectToLds for both unroll and tail loops - # - Tail loop can be unrolled up to InnerUnroll amount if AssertSummationElementMultiple%InnerUnroll==0 - # - # 1 indicates no assertion (since all sizes are multiples of 1) - "AssertSummationElementMultiple": [1, 2, 4, 8, 16, 32, 64, 128], - # Kernel generator will assume that the FreeIndex[0] size is some multiple of the element size - # and uses this to optimize the kernel. - # FreeIndex[0] is usually letter "I" - # (Recommended AF0EM value is 8 for half, 4 for single, 2 for double) - # - # Optimizations enabled by AssertFree0ElementMultiple>1: - # Load optimizations: - # - For TLU=1 matrix, if AF1WM>=GLVW then can enable UseSgprForGRO - # - Reduces registers used for address calculations - # - Removes address shift/unshift code - # - UseSgprForGRO will only be enabled if all matrices meet assertion requirements. - # - # Store Optimizations: - # - Can vectorize stores in edge tiles. Vector width can be up to AF0EM. - # (since C matrix is always coalesced in Free0 index direction and this assertion guarantees the index element multiple) - # - # 1 indicates no assertion (since all sizes are multiples of 1) - "AssertFree0ElementMultiple": [1, 2, 4, 8, 16], - # Kernel generator will assume that the FreeIndex[1] size is some multiple of the element size - # and uses this to optimize the kernel. - # FreeIndex[1] is usually letter "J" - # (Recommended AF1EM value is 8 for half, 4 for single, 2 for double) - # Optimizations enabled by AssertFree1ElementMultiple>1: - # - See above AssertFree0ElementMultiple "Load optimizations" - # 1 indicates no assertion (since all sizes are multiples of 1) - "AssertFree1ElementMultiple": [1, 2, 4, 8, 16], - # Assertions that require arithmetic intensity to be specified value. - # Arithmetic intensity measures the ratio of computation to memory bandwidth required for a problem. - # These predicates can be used to adjust solution selection compute-bound or memory-bound problems. - "AssertAIGreaterThanEqual": -1, - "AssertAILessThanEqual": -1, - # Stagger the start summation position of the tiles. - # Elements from the summation dimension are loaded at offsets rather than all starting at 0. - # StaggerU is the max 'clicks' of StaggerUStride bytes where each wg starts ; see StaggerUMapping - # for how the specific stagger for a given wg is determined. - # - # The tile assignment C are same as with StaggerOffset=0 ; the difference is the - # order that the summation elements are added. - # GRO will wrap back to the row start when the edge is reached. - # - # This can be effective for TLU=0 style matrices where the K dimension is a large power-of-2. - # In this case the start of each row of the tile is separated by an exact power-of-2 - # which causes poor dram, cache, and tlb behavior. V20 has 16 channels each 256 bytes wide. - # StaggerU adjusts the start position in the summation (aka 'U') dimension - # to avoid these conflicts. Both A and B matrix start at the adjusted position. - # If >0 specifies the offset in multiples of the macro-tile "unroll" dim - # - Higher values will spread traffic to more channels but provide less L2 re-use. - # - StaggerU and WorkGroupMapping interact and should be tuned together - - # The WGM controls how tiles are assigned in C matrix, while StaggerU controls where those - # tiles start reading their summation dim parms. - # - StaggerU requires BufferLoad==1 and is silently ignored if BufferLoad==0 - "StaggerU": [0, 2, 4, 8, 16, 32, 64], - # Stride in bytes for each staggeru 'click'. - # 256 is recommended since this is the width of memory channel (on gfx803,gfx900,gf906) - so - # each click will start in a new memory channel and spread traffic among the 16 available channels. - # For example StaggerUStride=256 and StaggerU=8 will use 8 unique starting points - # in summation dimension, each offset by 256-bytes - provided the tensor dims are large - # enough to support this. - # StaggerUStride will be internally increased so it is an integer multiple of DepthU*BpeAB. - # (the implementation requires this - the unroll iteration accesses data in steps of - # DepthU*BPE - "StaggerUStride": [-1, 16, 32, 64, 128, 256, 512, 1024, 2048], - # How the tile assignment (wg0, wg1, wg2) controls the initial StaggerU offset: - # 0: Use wg0 - # 1: Use wg1 - # 2: Use wg2 - # 3: Use wgSerial, wgSerial = wg0 + wg1 * nwg0 + wg2 * (nwg0 * nwg1) - # 4: Debug mode, offset each tile max allowed StaggerU. This just moves hotspot - # to a different bank since all workgroups still start at same point. - "StaggerUMapping": [0, 1, 2, 3, 4], - # GSU Workgroup Coalesced Ordering - # False: {(wg0,wg1,wg2,wgn)|(wg0,wg1,wg2,wgn)|...|(wg0,wg1,wg2,wgn)} - # True: {(wg0,wg0,wg0)|(wg1,wg1,wg1)|(wg2,wg2,wg2)|...|(wgn,wgn,wgn)} - "GlobalSplitUCoalesced": [False, True], - # GSU Workgroup Mapping - # False: wg issued order = {(wg0,wg1,wg2,wgn),(wg0,wg1,wg2,wgn)|...|(wg0,wg1,wg2,wgn)} - # -> workgroups do the summation by tile -> slower GR but faster GW - # True: wg issused oder = {(wg0,wg0,wg0)|(wg1,wg1,wg1)|(wg2,wg2,wg2)|...|(wgn,wgn,wgn)} - # -> workgroups split up the summation -> faster GR but slower GW - "GlobalSplitUWorkGroupMappingRoundRobin": [False, True], - # 0=don't use magic div (source only) - # 1=magic div alg #1. Slightly faster but limited range (if magic number is 2^32) - # 2=magic div alg#2. Slightly slower but handles all unsigned ints up to 2^32 - "MagicDivAlg": [0, 1, 2], - # For Block Mapping type: - # 0 : Use hardware-assigned wg number with no remapping. - # N : WG block width. "Wrap" to a new wg1 "row" assignment after N WGs assigned in that row. - # Tensor C always mapped with first free coord as fastest moving - # (Elements in this dimension are sequential in memory. - # - # For 2D nonbatched Matrix this means index order is I, then J - # For 2D batched Matrix this means index order is I, then J, then K. - # - # Then for 2D case: - # - If drawn in row-major format, I is the width and J is the height. - # - WGM determines dimensions of the box used to assign tiles from C - # - WGM is the height of the box (in the J dimension) - # - Given WGM, the box width (in I dim) is determined by number of CUs - # - The box always moves across matrixC in the fastest-moving "I" dim, then - # wraps to next J. TODO - might be useful to change this? - # - # Examples for 2D matrix: - # WGM=8: on CU64 machine this is a square box - # WGM=1: Short/Fat - this will cover maximum width in I dimension of C. This matches hardware assigned mapping. - # WGM=64: Tall/Skinny - this will cover maximum width in J dimension of C. - # - # Formula for wgSerial: - # wgSerial = wg0 + (wg1 % WorkGroupMapping) * nwg0 - "WorkGroupMapping": list( - range(-1024, 1024 + 1) - ), # change a workgroup's id so that the all the workgroups on the gpu at a time are hitting L2 cache the best - "WorkGroupMappingXCC": [ - 1, - 2, - 4, - 8, - 16, - 32, - ], # change a workgroup's id so that contiguous workgroup can map on same XCC - # -1 : WorkGroupMappingXCCGroup will be set to CU_count at runtime. Please ensure that (CU_count % WGMXCC == 0). - "WorkGroupMappingXCCGroup": list( - range(-1, 1024) - ), # change a workgroup's id so that contiguous workgroup can map on same XCC, remap workgroup in a group of WGMXCCG. - "MaxOccupancy": list( - range(1, 40 + 1) - ), # wg / CU; if cache thrashing is hurting performance, this allocates extra lds to artificially limit occupancy - "WorkGroup": validWorkGroups, # ( wg0 x wg1 x LocalSplitU ) dimensions of the workgroup which will operate on a tile and share lds - # ThreadTile: ( tt0 x tt1 ) dimensions of the C tile that each thread works on, - # TT=4 and VW=4 means a thread will work on a tight 4x4 tile of C, where VW=1 means the tile will work on 16 spread out values - # Generally, the VW determines the consecutive a WI will work on, then it will skip ahead SG0*VW elements to get to the next row of VGPR inputs - "ThreadTile": validThreadTiles, - "MacroTile": validMacroTiles, # MT0 = wg0*tt0, MT1 = wg1*tt1 - "WavefrontSize": [32, 64], - # MatrixInstruction: (M x N x K x B) - # XDLOPS tile definition, only valid for gfx908, gfx90a - # MxNxKxB specifies matrix instruction variants - # MxNxB determines the shape of the C tile each instruction worked on - # K determines the unroll depth - # If empty, do not use these instructions - # - # Alternative format: (M x N x K x B x MIBlockM x WaveTileM x WaveTileN x WaveM x WaveN) - # (Note: MxN means M-by-N in the following comments) - # MIBlockM determines how many blocks along M dimension for multi-block MI variants. Concrete examples: - # - MI 16x16x1x4 (4-block variant) with MIBlockM=4 -> (16x16)*(4x1)=64x16 tile per instruction executed - # - MI 32x32x1x2 (2-block variant) with MIBlockM=1 -> (32x32)*(1x2)=32x64 tile per instruction executed - # WaveTileM/N are dimensions of the C tile each wave works on, and is close to the concept of ThreadTile in classic VALU kernels - # - WT 4x1 -> each wave executes 4x1 matrix instructions on the C tile of total area (4*MITileM)x(1*MITileN) - # WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads - # - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2 - # Putting it all together: - # - [32, 32, 1, 2, 1, 4, 1, 2, 2] - # ^^^^^^^^^^^^ ^ ^^^^ ^^^^ - # MatrixInst BlkM WT Wave - # - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile - # Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used - "MatrixInstruction": validMatrixInstructions, - # StoreRemap: Optimize MatrixInstruction store patterns to enhance performance. - # MI output data between each threads are along N dims. - # But global memory is along M dim continuous. - # That mean global write between each threads are not continuous. - # Therefore, store performance for MI instruction is poor. - # How StoreRemap works in final store stage: - # 1. Put all thread output data into LDS. - # 2. All thread read data from LDS along M dims. - # (match global Memory continuous direction) - # 3. All thread write out data into global memory. - # 0: Disable StoreRemap (default) - # 1~8: Enable StoreRemap and set the global write vector width - # Suggest optimum value: fp32 = [2,4], fp16 or bf16 = [4,8] (dwordx2 and dowrdx4) - # -1: Use dwordx2 if support SRVW, or set SRVW to 0 - "StoreRemapVectorWidth": [-1, 0, 1, 2, 4, 8], - # SourceSwap: Optimizes MatrixInstruction store pattern by swapping mfma input order. - "SourceSwap": [False, True], - # Following parameters are designed for store scheduling. - # (store stands for load from C (with beta) and store to C/D) - # - # we want to hide store behind unroll loop - # 1. if we can launch 2 WorkGroups per CU (occupancy >= 2, large M/N) - # 2. if there are remaining global memory bandwidth in unroll loop (compute bound kernel) - # - # we can hide store behind the other WG's loop by lowering priority of store - # priority of loop is the same as priority of store - # WG0: ???????????????\__ - # |<-- loop --->|<-- store -->|end - # - # WG1: ___________________________/????????????\__ - # |<--------- loop ------------------->|<-- store -->|end - # - # priority of loop is higher than priority of store - # WG0: ???????\____________________ - # |<-- loop --->|<------ store ----->|end - # - # WG1: _____________/?????\__________________ - # |<------- loop -------->|<----- store ---->|end - "StorePriorityOpt": [False, True], - # - # If we issue store in short period of time, kernel will become from compute bound to memory bound - # 0 means issue instructions as many as possible if VGPR available - "NumElementsPerBatchStore": list(range(-1, 256)), - # - # add sync after per batch store in order to store contiguous elements - # add sleep after per batch store in order to distribute store over whole loops - # NOTE: this parameter is highly depends on size_k - # 0 means no sync and sleep - "StoreSyncOpt": list(range(0, 256)), - # - # There are index or address calculation between global instructions. - # issue global instruction b2b has better performance - "GroupLoadStore": [False, True], - # In order to remove the copying from Acc vgpr to Arch vgpr, only use Arch vgprs for v_mfma_xxx. - # Only support for kernel whose totalVgpr counts less than 256 and gcn that has control bit ACC_CD. - "MIArchVgpr": [False, True], - # StreamK (SK) kernels divide work evenly among CUs by splitting along MT and K dimensions. - # Total work units are calculated as (#MTs x #LoopIters) and divided among workgroups. - # In most cases each workgroup will calculate a partial tile that are accumulated in a fixup step in the same kernel - # 0 : Standard data-parallel kernel - # 1 : Basic StreamK - # 2 : Two-Tile StreamK (each WG completes an even number of sk iterations, followed by an even number of dp tiles) - # 3 : Two-Tile StreamK with DP before SK tiles - # StreamK kernels can adjust the number of CUs being used. - # Using fewer sometimes increases overall throughput by allowing other kernels to run in parallel. - # StreamK grid is controlled by setting these enviornment variables: - # TENSILE_STREAMK_FIXED_GRID lets you override the default grid size with a specific number - # 0 = override disabled (default) - # TENSILE_STREAMK_FULL_TILES sets the number of full tiles to be included in stream-k work - # -1 = use prediction model for best performance (not yet implemented) - # 0 = only remainder tiles run in stream-k - # 1+ = remainder + 1 (or more) full grids of tiles run in stream-k (default=1) - # TENSILE_STREAMK_DYNAMIC_GRID selects dynamic grid mode, which automatically limits the number of CUs used: - # 0 = Off, always use all CUs. - # 1 = Only reduce CUs for small problems to number of output tiles when num_tiles < CU count. - # 2 = Also reduce CUs used for large sizes to improve data-parallel portion and reduce power. - # 3 = Analytically predict the best grid-size by weighing the cost of the fix-up step and the cost of processing MACs (default). - # Note: dynamic grid coefficients currently apply to gfx942 variants - # TENSILE_STREAMK_MAX_CUS allows the user to manually set maximum number of CUs used, which could free up some CUs for - # other operations to run in parallel with gemm. - # TENSILE_STREAMK_GRID_MULTIPLIER lets you set how many workgroups are created per CU being used. - # 1 = 1 WG per CU (default), for example. 2 will launch WGs = 2 x CU count. - # The priority of these environment variables is defined as follows: - # TENSILE_STREAMK_FIXED_GRID > TENSILE_STREAMK_DYNAMIC_GRID > TENSILE_STREAMK_MAX_CUS > TENSILE_STREAMK_GRID_MULTIPLIER - "StreamK": [0, 1, 2, 3], - # Determines if StreamK kernel uses atomics - # 0: uses workspace to store partial tiles, accumulate in deterministic fix-up step - # 1: uses atomics to accumulate partial tiles - "StreamKAtomic": [0, 1], - # Enables XCC-based remapping of workgroups, set the value to the number of XCCs - # for the device/configuration being used - # 0: uses default workgroup assignment - # 2+: remaps workgroups to be contiguous within an XCC for a given number of XCCs - "StreamKXCCMapping": [0] + list(range(2, 9)), - # Debug settings for stream-k kernels to disable parts of the kernel - # Bit 0: Don't generate fixup code - # Bit 1: Don't generate write to partials code - # Both parts can be disabled together - # 0 = Debug mode off, generate full kernel - # 1 = No fixup - # 2 = No partials - # 3 = Nofixup and no partials - "DebugStreamK": [0, 1, 2, 3], - # Controls desired width (#elements) for loads from global memory -> LDS. - # and eliminates the pointer unshift logic - # -1 : Set GlobalReadVectorWidth = VectorWidth - # NOTE: for input bpe=32, max GRVW is 4 (to fit dwordx4) (FP32), min GRVW is 1 (dword) - # bpe=16, max GRVW is 8 (to fit dwordx4) (FP16), min GRVW is 2 (dword) - # bpe=8, max GRVW is 16 (to fit dwordx4) (INT8), min GRVW is 4 (dword) - "GlobalReadVectorWidthA": [-2, -1, 1, 2, 3, 4, 6, 8, 16], - "GlobalReadVectorWidthB": [-2, -1, 1, 2, 3, 4, 6, 8, 16], - # Controls desired width (#elements) for loads from LDS -> VGPR. - # -1 : Set LocalReadVectorWidth = VectorWidth - # 1 cannot be used for half type. - # used in combination with TransposeLDS=True - # in TransposeLDS=1 case, use wider load to fetch elements in summation dimension from LDS - # helps optimizing instruction scheduling between MFMA and nonMFMA instructions - # NOTE: for input bpe=32, max LRVW is 4 (to fit ds_read_b128) (FP32) - # bpe=16, max LRVW is 8 (to fit ds_read_b128) (FP16) - # bpe=8, max LRVW is 16 (to fit ds_read_b128) (INT8) - "LocalReadVectorWidth": [-1, 1, 2, 4, 8, 16], - # threads should read/write/operate on this many contiguous elements from the C matrix. - # If VW=4 then thread0 will process 4 consec C elements, then thread1 next 4, etc. - # If the ThreadTile is > VectorWidth then thread0 will next operate on the 4 elements in C at (4*NumThreads) - # Typically the load vector width and store vector width are directly related to the VW. - # The global load width is closely related to the width of local stores so - # GlobalReadVectorWidth also controls local write width. - # Local read width also matches since VectorWidth consec elements must be read - # Typically matching 16 bytes is good choice since the stores will be optimally coalesced with 16 bytes/WI. - # Using a VW too large which results in >16bytes/thread isn't supported - # For MFMA non SourceSwap: this parameter didn't take effect - # -1 means set vw to largest localReadWidth according to MIWaveTile - "VectorWidthA": [-1, 1, 2, 3, 4, 6, 8], - "VectorWidthB": [-1, 1, 2, 3, 4, 6, 8], - # If 0, store 1 element per instruction. - # If 1, store vector-width elements per instruction. - # if -1, store vector-wide elements per instruction unless PBD would not generate a valid kernel - "VectorStore": [-1, 0, 1], - # Controls desired width (#elements) for stores from reg to global memory. - # When MatrixInstruciton == None, derived parameter gwvw takes precedence. - # -1 : Set StoreVectorWidth = VectorWidth - "StoreVectorWidth": [-1, 1, 2, 3, 4, 6, 8], - # when loading all the data from global into lds requires multiple load instructions, these parameters govern which - # loads will pull which rectangle of data from global into lds - # NLC=1 means one load along the coalesced dimension, which results in the most coalescing possible - # NLC=-1 looks for the largest number of reads along the coalesced dimension which results in the least ammount of coalescing; - # however in this case the stride between one load and another is a static value, therefore buffer loads only need one set of registers - # whereas the =1 case has a stride which is a multiple of a kernel argument and therefore needs one address per load in the perpendicular dimension - "NumLoadsCoalescedA": list(range(-1, 64 + 1)), - "NumLoadsCoalescedB": list(range(-1, 64 + 1)), - # DepthU, LocalSplitU (which is the 3rd number in WorkGroup), and LoopUnroll are closely related - # LoopUnroll=4 means there are 4 subiterations within the loop, 4 actual iterations written in the code. - # LocalSplit=2 means the workgroup is split up into 2 subgroups, and each subgroup is doing different parts of the summation. - # subgroup0 does k=0-3, 8-11... and subgroup1 does k=4-7, 12-15... - # So, each iteration through the summation loop, which has 4 actual subiterations, does 8 summation iterations, because each subgroup did 4; - # and when data is read from global memory the threads read 8 elements along the summation dimension. - # DepthU = LoopUnroll * LocalSplitU = 4*2 in this case - # it made more sense for the user to directly control LocalSplitU and DepthU, then derrive afterwards LoopUnroll=DepthU/LocalSplitU - # -1 : Only allow GLVW=1 - # -2 : Only allow max(GLVWA,GLVWB) < VW ? - # -3 : Only allow min(GLVWA,GLVWB) < VW ? - "DepthU": depthUs, - # integer amount of padding to put into LDS, in 2016 this didn't seem to help performance, profilers were showing that channel conflicts weren't really hurting - # performance so this has been deprecated and probably doesn't work - # -1 means use same padding as the VectorWidth if TLU=0 else 0. (Padding only helps when transpose is required) - # With MatrixInstruciton: -1 means max(GRVW,MIInput) if TLU=0 - "LdsPadA": [-1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64], - "LdsPadB": [-1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64], - "LdsPadMetadata": [-1, 0, 1, 2, 3, 4, 8], - # Padding boundary for LDS. defines block-size for pad insertion. for every 'LdsBlockSizePerPad' bytes, LDS padding (pad value from LdsPad parameter) - # is added (readOffset aware of the pad and adjusts offset value based on this parameter value). - # Only support LdsBlockSizePerPad >= unrollDepth * BPE - # 0 means disable LdsBlockSizePerPad - "LdsBlockSizePerPadA": [-1, 0, 64, 128, 256, 512, 1024, 2048], - "LdsBlockSizePerPadB": [-1, 0, 64, 128, 256, 512, 1024, 2048], - "LdsBlockSizePerPadMetadata": [-1, 0, 64, 128, 256, 512, 1024, 2048], - # Transpose LDS format. Local store in coalesced dimension , same as optimized global fetch dimension . applicable only in TLU=0 case for miSIMD(s) - # -1 : keep LDS layout same as global fetch dimension for both A and B - # set TLDS = 1 for NN,TN,TT - # set TLDS = 0 for NT - # 0 : coalesced dimension of lds is tile dimension - # 1 : keep LDS layout same as global fetch dimension for both A and B for NN,TN,TT, but NT would be rejected - # 2 : coalesced dimension of lds is unroll dimension for both A and B - "TransposeLDS": [-1, 1, 0, 2], - # add gls or slc after global memory read/writes to change caching, not caching the writes is promising and improved performance a tiny bit - # 0: none, 1: glc, 2: slc, 3: glc slc - # For gfx942, sets sc0/sc1/nt bits - # 0: none, 1: sc0, 2: sc1, 3: sc0 sc1, 4: nt, 5: nt sc0, 6: nt sc1, 7: nt sc0 sc1 - "NonTemporalE": list(range(0, 8)), - "NonTemporalD": list(range(0, 8)), - "NonTemporalC": list(range(0, 8)), - "NonTemporalA": list(range(0, 8)), - "NonTemporalB": list(range(0, 8)), - "NonTemporalWS": list(range(0, 8)), - "NonTemporalMetadata": list(range(0, 8)), - "NonTemporal": list(range(-1, 8)), - # Group together unroll iterations inside the unroll loop. - # For example, InnerUnroll=2 will fetch LDS for two unroll iterations - "InnerUnroll": [1, 2, 4, 8, 16, 32, 64], - # Enable CP preload kernel arguments feature - # It can reduce time of loading kernel arguments by s_load. - # It needs new complier and vbios to support this feature. - "PreloadKernArgs": [False, True], - # Kernels should be written in assembly or source - # if assembly, ISA will determine architecture - # if source, Runtime will determine language - # later on, we'll relax this to inner kernel languages and outer kernel languages, such as inline asm embedded in ocl or in llvm - "KernelLanguage": ["Assembly"], - "ISA": validISA, # arch for assembly kernels - # Name of the custom kernel located at `CUSTOM_KERNEL_PATH`. - # a custom kernel is a user written assembly kernel with its associated configuration parameters included in a custom.config section - # inside the yaml block between the --- and ... markers. These parameters are only used for information purposes, not kernel generation. - # Ex: - # custom.config: - # ProblemType: - # OperationType: GEMM - # etc... - # ThreadTile: [8, 8] - # etc... - # - # Custom kernels can be included in a BenchmarkProblemSizeGroup by having their name (without file extension) listed under the "CustomKernels" - # category alongside InitialSolutionParameters, BenchmarkCommonParameters, etc... - "CustomKernelName": -1, - # Will allow a kernel to be accepted even when checks determine it's not viable. - # Intended for use with custom kernels which have confirmed to be correct - "NoReject": [False, True], - # Debug use only. - "ActivationFused": [False, True], - # True- function call - # False- inline - "ActivationFuncCall": [False, True], - # Alternative implementation for activation function - # Currently only supports GSU == 1 - "ActivationAlt": [False, True], - # Do workgroup reduction. Currently for DBias - "WorkGroupReduction": [False], - # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B - "Sparse": [0, 1, 2], - # in mix mode F8 need to convert to F16, do this before(0) ds or after(1) ds - "ConvertAfterDS": [False, True], - # Force disable shadow init to release more sgpr in preloop - "ForceDisableShadowInit": [False, True], -} - - # same parameter for all solution b/c depends only on compiler defaultBenchmarkCommonParameters = [ {"InnerUnroll": [1]}, @@ -1363,97 +438,7 @@ defaultSolution[key] = value[0] # other non-benchmark options for solutions -################################################################################ -# Default Problem Type -################################################################################ -defaultProblemType = { - # =GEMM uses TransposeA,B parameters and makes the problem type more readable for users - # =TensorContraction requires specifying - "OperationType": "GEMM", # GEMM, TensorContraction, ConvolutionForward, ConvolutionBackwardData, ConvolutionBackwardWeights - "DataType": 0, # data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeA": 0, # A data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeB": 0, # B data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeE": 0, # E data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DataTypeAmaxD": 0, # AmaxD data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "DestDataType": 0, # destination data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "ComputeDataType": 0, # compute data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType - "F32XdlMathOp": 0, # reducing intermediate precision from f32 to a specific type, such as "x", as listed in SolutionStructs.py::DataType. - # in:f32, intermediate:xf32, out:f32. f32 = xf32(f32) * xf32(f32) - "UseBeta": True, # =True use beta parameter (asm will check for B=0 and optimize the write for that), =False don't use beta parameter - "UseE": False, # =True use output E to output gemm results before activation - "Gradient": False, # =True set globalWriteElements to gradient mode - "UseBias": 0, # =1 support bias vector on M direction, =2 support bias vector on N direction, =3 support bias vector on both M,N direction - "BiasSrc": "D", # This parameter is used in gradient + bias. Support A, B, D. - "UseScaleAB": "", # Support "", "Scalar", and "Vector" - "UseScaleCD": False, # =True use scaleC, scaleD - "UseScaleAlphaVec": 0, # =1 support alpha vector on M direction, =2 support bias vector on N direction, =3 support alpha vector on both M,N direction - "HighPrecisionAccumulate": False, # f32 += f16*f16 - "SilentHighPrecisionAccumulate": False, # Keep kernel names the same for HPA mode. Useful for testing. - "Sparse": 0, # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B - "ComplexConjugateA": False, # complex data should be conjugated for "C" transpose case - "ComplexConjugateB": False, - "StochasticRounding": False, # By default, IEEE RNE rounding - # for OperationType == GEMM - "TransposeA": False, # =True means transA="T" or "C", =False means transA = "N" - "TransposeB": True, - "Batched": False, # add batching dimension - "StridedBatched": True, # use to select general batch or strided batch - "GroupedGemm": False, # use to select general batch or strided batch - # for OperationType == TensorContraction - # - Indices < NumIndicesC are Free or Batch indices and appear in C and D - # - Indices which appear in both A and B, and are < NumIndicesC are batch. A and B must have same number of batch indices. - # - Indices which appear in both A and B, and are >= NumIndicesC are summation. A and B must have same number of summation indices. - # - Indices which appear in A or B (but not both), are Free. A and B may have different numbers of free indices. - # - Summation loops are nested from smallest index number to largest, with the largest summation index as the 'unroll' loop. - # - Memory order of C and D matrices is always 0..NumIndicesC-1, with 0 as the fastest-moving. - # - By choosing index assignments the output can be 'transposed'. For example if IA=[1,2] IB=[0,2] then 0 is the coalesced dim for C/D. - # - Likewise batch index may be assigned between two free indices to control the output order, ie to write in CNHW format. - # - For example : IA=[0,1,3] IB=[2,1,3]. 0,2 are free indices; 1 is batch. - "IndexAssignmentsA": [0, 2], - "IndexAssignmentsB": [1, 2], - "NumIndicesC": 2, - # use initial strides for AB. - # This has some performance impact for the increased flexibility: - # - Additional strides will be passed into the kernel and will occupy SGPR registers - # - GlobalReadWidth must be 1 (since elements are not guaranteed to be adjacent in memory) - "UseInitialStridesAB": False, - # use initial strides for CD. - # This has some performance impact for the increased flexibility: - # - Additional strides will be passed into the kernel and will occupy SGPR registers - # - Additional multiply on the store address path - # -VectorStore must be 0. If VectorStore is -1, it will be silently set to 0 internally. - "UseInitialStridesCD": False, - "AllowNoFreeDims": False, # allow A or B to specify no free dims - # (if false, A and B must have at least one free dim) - # (if true, A and B must have at least one free or batch dim) - # SetConstStride* sets the specified stride in the problem. - # These no longer generate predicates - see AssertStrideEqualA/B below - # List of pairs of [index, constValue]. - # Index is a member of the global index assignments (not an offset into IndexAssignmentsA/B) - # EX: SetConstStrideA: [ [3, 1], [2, 4] ] sets - # strideA for index3 to constant '1' and stride for index2 to constant '4'. - "SetConstStrideA": [], - "SetConstStrideB": [], - "SetConstStrideBias": [], - # Summation dimension indices - "MirrorDimsA": [], - "MirrorDimsB": [], - "MirrorDimsMetadata": [], - # for LD description - "NumIndicesLD": 4, - "IndexAssignmentsLD": [3, 4, 5, 6], # order is LDD, LDC, LDA, LDB - # Tile aware solution selection - "TileAwareSelection": False, - # Activation - "Activation": False, - "ActivationNoGuard": False, - # AmaxD - "OutputAmaxD": False, - # For kernels putting arguments in workspaces instead of kernel arguments, they can choose to support user arguments input instead. - "SupportUserArgs": True, - "SwizzleTensorA": False, - "SwizzleTensorB": False, -} + defaultProblemSizes = [{"Range": [[2880], 0, 0]}] defaultBenchmarkFinalProblemSizes = [{"Range": [[64, 64, 64, 512], 0, 0]}] @@ -1476,86 +461,11 @@ } -################################################################################ -# Printing -# 0 - user wants no printing -# 1 - user wants limited prints -# 2 - user wants full prints -################################################################################ -def print1(message): - if globalParameters["PrintLevel"] >= 1: - print(message) - sys.stdout.flush() - - -def print2(message): - if globalParameters["PrintLevel"] >= 2: - print(message) - sys.stdout.flush() - - -def printWarning(message): - print("Tensile::WARNING: %s" % message) - sys.stdout.flush() - - -def printExit(message): - print("Tensile::FATAL: %s" % message) - sys.stdout.flush() - sys.exit(-1) - - ################################################################################ # Is query version compatible with current version # a yaml file is compatible with tensile if # tensile.major == yaml.major and tensile.minor.step > yaml.minor.step ################################################################################ -def detectGlobalCurrentISA_(detectionTool): - """ - Returns returncode if detection failure - """ - global globalParameters - - if globalParameters["CurrentISA"] == (0, 0, 0) and detectionTool: - process = subprocess.run([detectionTool], stdout=subprocess.PIPE) - if os.name == "nt": - line = "" - for line_in in process.stdout.decode().splitlines(): - if "gcnArchName" in line_in: - line += line_in.split()[1] - break # detemine if hipinfo will support multiple arch - arch = gfxToIsa(line.strip()) - if arch is not None: - if arch in globalParameters["SupportedISA"]: - print1("# Detected local GPU with ISA: " + isaToGfx(arch)) - globalParameters["CurrentISA"] = arch - else: - archList = [] - for line in process.stdout.decode().split("\n"): - arch = gfxToIsa(line.strip()) - if arch is not None: - if arch in globalParameters["SupportedISA"]: - print1("# Detected local GPU with ISA: " + isaToGfx(arch)) - archList.append(arch) - if len(archList) > 0: - globalParameters["CurrentISA"] = archList[globalParameters["Device"]] - if process.returncode: - printWarning("%s exited with code %u" % (detectionTool, process.returncode)) - return process.returncode - return 0 - - -def detectGlobalCurrentISA(): - """ - Returns returncode if detection failure - """ - errorCode = detectGlobalCurrentISA_(globalParameters["AMDGPUArchPath"]) - if errorCode: - printWarning("Attempting to detect ISA with rocm_agent_enumerator") - return detectGlobalCurrentISA_(globalParameters["ROCmAgentEnumeratorPath"]) - return errorCode - - def restoreDefaultGlobalParameters(): """ Restores `globalParameters` back to defaults. @@ -1569,51 +479,52 @@ def restoreDefaultGlobalParameters(): globalParameters[key] = value -def printTable(rows): - rows = list([[str(cell) for cell in row] for row in rows]) - colWidths = list([max([len(cell) for cell in col]) for col in zip(*rows)]) - - for row in rows: - for width, cell in zip(colWidths, row): - pad = " " * (width - len(cell)) - print(pad, cell, sep="", end=" ") - print() +# hopefully the isaInfoMap keys only contain isas we plan to build and not all +def printCapabilitiesTable(isaInfoMap: Dict[str, IsaInfo]): + """ + Prints a capability table for the given parameters and ISA information map. + Args: + supportedIsas: The ISAs to show in the table. + isaInfoMap: The ISA information map containing assembler and architecture capabilities. + """ -def printCapTable(parameters): - import itertools + def printTable(rows): + rows = [[str(cell) for cell in row] for row in rows] + colWidths = [max(len(cell) for cell in col) for col in zip(*rows)] - archs = [(0, 0, 0)] + parameters["SupportedISA"] - gfxNames = list(map(isaToGfx, archs)) + for row in rows: + print(" ".join(cell.ljust(width) for cell, width in zip(row, colWidths))) - headerRow = ["cap"] + gfxNames + def capRow(isaInfoMap, cap, capType): + return [cap] + [ + "1" if cap in getattr(info, capType) and getattr(info, capType)[cap] else "-" + for info in isaInfoMap.values() + ] - def capRow(caps, cap): - return [cap] + [("1" if cap in caps[arch] and caps[arch][cap] else "0") for arch in archs] + gfxs = list(map(isaToGfx, isaInfoMap.keys())) + headerRow = ["Capability"] + gfxs - allAsmCaps = set( - itertools.chain(*[caps.keys() for arch, caps in parameters["AsmCaps"].items()]) + allAsmCaps = sorted( + set(itertools.chain(*[info.asmCaps for info in isaInfoMap.values()])), + key=lambda k: (k.split("_")[-1], k), ) - allAsmCaps = sorted(allAsmCaps, key=lambda k: (k.split("_")[-1], k)) - asmCapRows = [capRow(parameters["AsmCaps"], cap) for cap in allAsmCaps] + asmCapRows = [capRow(isaInfoMap, cap, "asmCaps") for cap in allAsmCaps] - allArchCaps = set( - itertools.chain(*[caps.keys() for arch, caps in parameters["ArchCaps"].items()]) - ) - allArchCaps = sorted(allArchCaps) - archCapRows = [capRow(parameters["ArchCaps"], cap) for cap in allArchCaps] + allArchCaps = sorted(set(itertools.chain(*[info.archCaps for info in isaInfoMap.values()]))) + archCapRows = [capRow(isaInfoMap, cap, "archCaps") for cap in allArchCaps] printTable([headerRow] + asmCapRows + archCapRows) -def assignGlobalParameters(config, cxxCompiler=None): +def assignGlobalParameters(config, isaInfoMap: Dict[IsaVersion, IsaInfo]): """ Assign Global Parameters Each global parameter has a default parameter, and the user can override them, those overridings happen here """ - global globalParameters + global globalParameters, SUPPORTED_ISA # Minimum Required Version if "MinimumRequiredVersion" in config: @@ -1673,10 +584,6 @@ def assignGlobalParameters(config, cxxCompiler=None): os.path.join(globalParameters["ROCmPath"], "llvm/bin"), "ld.lld" ) - globalParameters["ExtractKernelPath"] = locateExe( - os.path.join(globalParameters["ROCmPath"], "hip/bin"), "extractkernel" - ) - if "AMDGPUArchPath" in config: globalParameters["AMDGPUArchPath"] = config["AMDGPUArchPath"] @@ -1689,39 +596,11 @@ def assignGlobalParameters(config, cxxCompiler=None): if "CodeObjectVersion" in config: globalParameters["CodeObjectVersion"] = config["CodeObjectVersion"] - # read current gfx version - returncode = detectGlobalCurrentISA() - if globalParameters["CurrentISA"] == (0, 0, 0): - printWarning( - "Did not detect SupportedISA: %s; cannot benchmark assembly kernels." - % globalParameters["SupportedISA"] - ) - if returncode: - if os.name == "nt": - globalParameters["CurrentISA"] = (9, 0, 6) - printWarning("Failed to detect ISA so forcing (gfx906) on windows") - - globalParameters["AsmCaps"] = {} - globalParameters["ArchCaps"] = {} - globalParameters["AsmBugs"] = {} - - for v in globalParameters["SupportedISA"] + [(0, 0, 0)]: - globalParameters["AsmCaps"][v] = initAsmCaps(v, cxxCompiler, False) - globalParameters["ArchCaps"][v] = initArchCaps(v) - globalParameters["AsmBugs"][v] = initAsmBugs(globalParameters["AsmCaps"][v]) - - if globalParameters["PrintLevel"] >= 1: - printCapTable(globalParameters) - - globalParameters["SupportedISA"] = list( - [ - i - for i in globalParameters["SupportedISA"] - if globalParameters["AsmCaps"][i]["SupportedISA"] - ] - ) + if verbosity >= 1: + printCapabilitiesTable(isaInfoMap) - validParameters["ISA"] = [(0, 0, 0), *globalParameters["SupportedISA"]] + isaList = list(isaInfoMap.keys()) + validParameters["ISA"] = [IsaVersion(0, 0, 0), *isaList] # For ubuntu platforms, call dpkg to grep the version of hip-clang. This check is platform specific, and in the future # additional support for yum, dnf zypper may need to be added. On these other platforms, the default version of @@ -1749,6 +628,10 @@ def assignGlobalParameters(config, cxxCompiler=None): # The following keys may be present in the config, but are not (or no longer) global parameters. ignoreKeys = [ + "Architecture", + "ShortNames", + "PrintLevel", + "Device", "UseCompression", "CxxCompiler", "CCompiler", @@ -1780,38 +663,4 @@ def restoreClocks(): atexit.register(restoreClocks) - setupRestoreClocks() - - -def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, defaultDictionary): - if key in sourceDictionary: - destinationDictionary[key] = deepcopy(sourceDictionary[key]) - else: - destinationDictionary[key] = deepcopy(defaultDictionary[key]) - - -def checkParametersAreValid(param, validParams): - """Ensures paramaters in params exist and have valid values as specified by validParames""" - (name, values) = param - if name == "ProblemSizes": - return - elif name == "InternalSupportParams": - return - - if name not in validParams: - printExit( - "Invalid parameter name: {}\nValid parameters are {}.".format( - name, sorted(validParameters.keys()) - ) - ) - - for value in values: - if validParams[name] != -1 and value not in validParams[name]: - msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}." - msgExt = ( - " (only first 32 combos printed)\nRefer to Common.py for more info" - if len(validParams[name]) > 32 - else "" - ) - printExit(msgBase.format(name, value, name, validParams[name][:32], msgExt)) diff --git a/tensilelite/Tensile/Common/Types.py b/tensilelite/Tensile/Common/Types.py index ddb6900e3d..f36e0777a5 100644 --- a/tensilelite/Tensile/Common/Types.py +++ b/tensilelite/Tensile/Common/Types.py @@ -1,9 +1,121 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +from dataclasses import dataclass from typing import NamedTuple, Tuple -IsaVersion = Tuple[int, int, int] +@dataclass +class IsaInfo: + asmCaps: dict + archCaps: dict + regCaps: dict + asmBugs: dict class SemanticVersion(NamedTuple): major: int minor: int patch: int + +IsaVersion = SemanticVersion + + +class DepthUConfig(NamedTuple): + deviceLDS: int=65536 + maxLDS: int=65536 + + +def makeDepthUConfig(config: dict) -> DepthUConfig: + deviceLDS = maxLDS = 65536 + if "DeviceLDS" in config: + deviceLDS = config["DeviceLDS"] + if "MaxLDS" in config: + maxLDS = config["MaxLDS"] + return DepthUConfig(deviceLDS, maxLDS) + + +class DebugConfig(NamedTuple): + enableAsserts: bool=False + enableDebugA: bool=False + enableDebugB: bool=False + enableDebugC: bool=False + expectedValueC: float=16.0 + forceCExpectedValue: bool=False + debugKernel: bool=False + forceGenerateKernel: bool=False + printSolutionRejectionReason: bool=False + splitGSU: bool=False + printIndexAssignmentInfo: bool=False + + +def makeDebugConfig(config: dict) -> DebugConfig: + + enableAsserts = False + enableDebugA = False + enableDebugB = False + enableDebugC = False + expectedValueC = 16.0 + forceCExpectedValue = False + debugKernel = False + forceGenerateKernel = False + printSolutionRejectionReason = False + splitGSU = False + printIndexAssignmentInfo = False + + if "EnableAsserts" in config: + enableAsserts = config["EnableAsserts"] + if "EnableDebugA" in config: + enableDebugA = config["EnableDebugA"] + if "EnableDebugB" in config: + enableDebugB = config["EnableDebugB"] + if "EnableDebugC" in config: + enableDebugC = config["EnableDebugC"] + if "ExpectedValueC" in config: + expectedValueC = config["ExpectedValueC"] + if "ForceCExpectedValue" in config: + forceCExpectedValue = config["ForceCExpectedValue"] + if "DebugKernel" in config: + debugKernel = config["DebugKernel"] + if "ForceGenerateKernel" in config: + forceGenerateKernel = config["ForceGenerateKernel"] + if "PrintSolutionRejectionReason" in config: + printSolutionRejectionReason = config["PrintSolutionRejectionReason"] + if "SplitGSU" in config: + splitGSU = config["SplitGSU"] + if "PrintIndexAssignmentInfo" in config: + printIndexAssignmentInfo = config["PrintIndexAssignmentInfo"] + + return DebugConfig( + enableAsserts, + enableDebugA, + enableDebugB, + enableDebugC, + expectedValueC, + forceCExpectedValue, + debugKernel, + forceGenerateKernel, + printSolutionRejectionReason, + splitGSU, + printIndexAssignmentInfo, + ) diff --git a/tensilelite/Tensile/Common/Utilities.py b/tensilelite/Tensile/Common/Utilities.py index 594317f564..f5679d5162 100644 --- a/tensilelite/Tensile/Common/Utilities.py +++ b/tensilelite/Tensile/Common/Utilities.py @@ -1,16 +1,69 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + import functools import math import os -import re import sys import time + +from inspect import currentframe, getframeinfo +from copy import deepcopy from enum import Enum -from typing import List, Tuple +from pathlib import Path from Tensile import __version__ -from .Architectures import isaToGfx +verbosity = 1 +################################################################################ +# Printing +# 0 - user wants no printing +# 1 - user wants limited prints +# 2 - user wants full prints +################################################################################ +def print1(message): + if verbosity >= 1: + print(message) + sys.stdout.flush() + + +def print2(message): + if verbosity >= 2: + print(message) + sys.stdout.flush() + + +def printWarning(message): + print("Tensile::WARNING: %s" % message) + sys.stdout.flush() + + +def printExit(message): + print("Tensile::FATAL: %s" % message) + sys.stdout.flush() + sys.exit(-1) # get param values from structures. def hasParam(name, structure): @@ -39,62 +92,8 @@ def locateExe(defaultPath, exeName): # /opt/rocm/bin, hip-clang exePath = os.path.join(path, exeName) if isExe(exePath): return exePath - return None - -def splitArchs(params: dict, fromTensile=False) -> Tuple[List[str], List[str]]: - """ - Splits and processes the architecture strings based on the provided parameters. - - Args: - params: A dictionary of global parameters. - fromTensile: A flag indicating if the function is called from the context of Tensile. - - Returns: - A tuple containing two lists: - - archs: A list of architecture strings with ``-`` instead of ``:`` - - cmdlineArchs: A list of architecture strings that retain ``:`` characters. - """ - - def isSupported(arch): - return ( - params["AsmCaps"][arch]["SupportedISA"] and params["AsmCaps"][arch]["SupportedSource"] - ) - - if ";" in params["Architecture"]: - wantedArchs = params["Architecture"].split(";") - else: - wantedArchs = params["Architecture"].split("_") - archs = [] - cmdlineArchs = [] - if "all" in wantedArchs: - for arch in params["SupportedISA"]: - if isSupported(arch): - if arch in [(9, 0, 6), (9, 0, 8), (9, 0, 10), (9, 4, 2)]: - if arch == (9, 0, 10): - archs += [isaToGfx(arch) + "-xnack+"] - cmdlineArchs += [isaToGfx(arch) + ":xnack+"] - if params["AsanBuild"]: - archs += [isaToGfx(arch) + "-xnack+"] - cmdlineArchs += [isaToGfx(arch) + ":xnack+"] - else: - archs += [isaToGfx(arch) + "-xnack-"] - cmdlineArchs += [isaToGfx(arch) + ":xnack-"] - else: - archs += [isaToGfx(arch)] - cmdlineArchs += [isaToGfx(arch)] - else: - for arch in wantedArchs: - archs += [re.sub(":", "-", arch)] - cmdlineArchs += [arch] - - # if calling from the context of Tensile we only want the arch associated with the current ISA - if fromTensile: - gfx = isaToGfx(params["CurrentISA"]) - archs = set(a for a in archs if gfx in a) - cmdlineArchs = set(a for a in cmdlineArchs if gfx in a) - - return archs, cmdlineArchs + raise OSError(f"Failed to locate {exeName}") def ensurePath(path): @@ -103,7 +102,7 @@ def ensurePath(path): except FileExistsError: pass except OSError: - printExit('Failed to create directory "%s" ' % (path)) + raise OSError('Failed to create directory "%s" ' % (path)) return path @@ -111,6 +110,14 @@ def roundUp(f): return (int)(math.ceil(f)) +def elineno(): + """ + Return the file name and line number of the caller. + """ + frame = getframeinfo(currentframe().f_back) + return f"{Path(frame.filename).name}:{frame.lineno}" + + ################################################################################ # Is query version compatible with current version # a yaml file is compatible with tensile if @@ -281,6 +288,7 @@ def hash_objs(*objs, **kwargs): return hash(tuple(objs)) +# Is this used? def ClientExecutionLock(lockPath: str): if not lockPath: return open(os.devnull) @@ -288,3 +296,10 @@ def ClientExecutionLock(lockPath: str): import filelock return filelock.FileLock(lockPath) + + +def assignParameterWithDefault(destinationDictionary, key, sourceDictionary, defaultDictionary): + if key in sourceDictionary: + destinationDictionary[key] = deepcopy(sourceDictionary[key]) + else: + destinationDictionary[key] = deepcopy(defaultDictionary[key]) diff --git a/tensilelite/Tensile/Common/ValidParameters.py b/tensilelite/Tensile/Common/ValidParameters.py new file mode 100644 index 0000000000..224a9cfe5f --- /dev/null +++ b/tensilelite/Tensile/Common/ValidParameters.py @@ -0,0 +1,838 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +import math + +from .Architectures import SUPPORTED_ISA + +################################################################################ +# Enumerate Valid Solution Parameters +################################################################################ +validWorkGroups = [] +for numThreads in range(32, 1025, 32): + for nsg in [1, 2, 4, 8, 16, 32, 64, 96, 128, 256]: + for sg0 in range(1, numThreads // nsg + 1): + sg1 = numThreads // nsg // sg0 + if sg0 * sg1 * nsg == numThreads: + workGroup = [sg0, sg1, nsg] + validWorkGroups.append(workGroup) + +validThreadTileSides = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + list( + range(20, 256, 4) +) +validThreadTiles = [] +for i in validThreadTileSides: + for j in validThreadTileSides: + validThreadTiles.append([i, j]) + +validActivationFormats = ("NCHW", "NHWC", "CNHW", "NCDHW", "NDHWC", "CNDHW") +validWeightFormats = ("KCYX", "KYXC", "CKYX", "CYXK", "KCZYX", "CKZYX", "CZYXK") +validMacroTileSides = [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1024, + 6, + 12, + 24, + 48, + 96, + 192, + 384, + 768, +] +validMacroTiles = [] +validISA = [(0, 0, 0)] +validISA.extend(SUPPORTED_ISA) +depthUs = list(range(2, 1024 + 1, 1)) +for i in validMacroTileSides: + for j in validMacroTileSides: + validMacroTiles.append([i, j]) + +validMFMA = {} +validMFMA["H"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] +validMFMA["S"] = [[32, 32, 1, 2], [32, 32, 2, 1], [16, 16, 1, 4], [16, 16, 4, 1], [4, 4, 1, 16]] +validMFMA["B"] = [[32, 32, 2, 2], [32, 32, 4, 1], [16, 16, 2, 4], [16, 16, 8, 1], [4, 4, 2, 16]] +validMFMA["4xi8"] = [ + [32, 32, 4, 2], + [32, 32, 8, 1], + [16, 16, 4, 4], + [16, 16, 16, 1], + [4, 4, 4, 16], + [32, 32, 16, 1], + [16, 16, 32, 1], +] +validMFMA["D"] = [[16, 16, 4, 1], [4, 4, 4, 4]] +validMFMA["B1k"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] +validMFMA["C"] = validMFMA["S"] +validMFMA["Z"] = validMFMA["D"] +validMFMA["I8"] = [ + [32, 32, 4, 2], + [32, 32, 8, 1], + [16, 16, 4, 4], + [16, 16, 16, 1], + [4, 4, 4, 16], +] + [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1]] +validMFMA["F8"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["B8"] = validMFMA["F8"] +validMFMA["F8B8"] = validMFMA["F8"] +validMFMA["B8F8"] = validMFMA["F8"] +validMFMA["F8N"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validMFMA["B8N"] = validMFMA["F8N"] +validMFMA["F8B8N"] = validMFMA["F8N"] +validMFMA["B8F8N"] = validMFMA["F8N"] +validWMMA = [ + [16, 16, 16, 1], +] +validTT = 32 +validMFMA["_format9"] = [] + +for MFMA in [ + validMFMA["H"], + validMFMA["S"], + validMFMA["B"], + validMFMA["D"], + validMFMA["X"], + validMFMA["F8N"], + validWMMA, +]: + for MI in MFMA: + for bm in range(int(math.log(MI[3], 2)) + 1): + for tt0 in range(1, validTT + 1): + for tt1 in range(1, validTT + 1): + for wave_m in range(3): + for wave_n in range(3): + validMFMA["_format9"].append( + [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] + ) +validMatrixInstructions = ( + [[], [-1]] + + validMFMA["H"] + + validMFMA["S"] + + validMFMA["B"] + + validMFMA["D"] + + validMFMA["B1k"] + + validMFMA["X"] +) +validMatrixInstructions = validMatrixInstructions + validMFMA["_format9"] + +validSMFMA = {} +validSMFMA["H"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validSMFMA["B"] = [[32, 32, 16, 1], [16, 16, 32, 1]] +validSMFMA["4xi8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] +validSMFMA["I8"] = validSMFMA["4xi8"] +validSMFMA["F8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] +validSMFMA["B8"] = validSMFMA["F8"] +validSMFMA["F8B8"] = validSMFMA["F8"] +validSMFMA["B8F8"] = validSMFMA["F8"] +validSMFMA["F8N"] = [[32, 32, 32, 1], [16, 16, 64, 1]] +validSMFMA["B8N"] = validSMFMA["F8N"] +validSMFMA["F8B8N"] = validSMFMA["F8N"] +validSMFMA["B8F8N"] = validSMFMA["F8N"] +validSMFMA["_format9"] = [] +for SMFMA in [validSMFMA["H"], validSMFMA["B"], validSMFMA["4xi8"], validSMFMA["F8N"]]: + for MI in SMFMA: + for bm in range(int(math.log(MI[3], 2)) + 1): + for tt0 in range(1, validTT + 1): + for tt1 in range(1, validTT + 1): + for wave_m in range(3): + for wave_n in range(3): + validSMFMA["_format9"].append( + [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] + ) +validSparseMatrixInstructions = validSMFMA["H"] + validSMFMA["B"] + validSMFMA["4xi8"] +validMatrixInstructions = ( + validMatrixInstructions + validSparseMatrixInstructions + validSMFMA["_format9"] +) + + +validParameters = { + # 0: Global read is along parallel direction in thread level, + # each load instruction stride whole threads. + # ----> perp + # | [w0, w0, w1,w1,w2,w2,w3,w3, w0, w0, w1,w1,w2,w2,w3,w3] + # | [ t0,t32] [ ] [ t0,t32] [ ] + # para | [ t1,t33] [ wave 1,2,3 ] [ t1,t33] [ wave 1,2,3 ] + # | [ .., ..] [ ] [ .., ..] [ ] + # | [t31,t63] [ ] [t31,t63] [ ] + # V [-load_1] [-load_2] + # + # 1: Each wave load a block of memory, + # each load instruction stride 64 threads. + # ----> perp + # [ w0, w0, w0, w0, w1,w1,w1,w1, w2,w2,w2,w2, w3,w3,w3,w3] + # | [ t0,t32][ t0,t32] + # para | [ t1,t33][ t1,t33] + # | [ .., ..][ .., ..] + # | [t31,t63][t31,t63] + # V [-load_1][-load_2] + # + # + # 2: Each load instruction spread threads evenly in the perp direction + # ----> perp + # | [w0, w1, w2, w3, w0, w1, w2, w3, w0, w1, w2, w3, w0, w1, w2, w3] + # | [t0 ] [t0 ] [t32] [t32] + # para | [t1 ] [t1 ] [t33] [t33] + # | [.. ] [.. ] [.. ] [.. ] + # | [t31] [t31] [t63] [t63] + # V [load_1] [load_2] [load_1] [load_2] + # + "WaveSeparateGlobalReadA": [0, 1, 2], + "WaveSeparateGlobalReadB": [0, 1, 2], + # Add an unrolled loop and NGLL loop with swapped GRA and GRB order. + # which may change the tlb thrashing behavior. + "UnrollLoopSwapGlobalReadOrder": [0, 1], + # PrefetchGlobalRead = 1: + # Requires 2X LDS space, and VGPRs for buffering data on way into LDS + # prefetch / double-buffer reads from global memory -> vgprs -> lds. + # + # PrefetchGlobalRead = 2: + # Do another prefetch while writing data from vgpr to lds. + # prefetch / double-buffer reads from global memory -> vgprs --> lds. + # |-> prefetch reads + "PrefetchGlobalRead": [0, 1, 2], + # number of iteration prefetch local reads from lds to VGPRs buffer = PLR + "PrefetchLocalRead": list(range(128 + 1)), + # MatrixInstruction Only + # If set ClusterLocalRead, each iteration dedicated vgprBuffer for localRead + # So we can schedule these localReads to the front of the loop + "ClusterLocalRead": [0, 1], + # We use double LDS buffer when PrefetchGlobalRead. + # While it reads data from LDS[0]/[1], it prefetch global data and writes to LDS[1]/[0] + # If we can make sure all data are read from LDS to register before writing data to LDS, we can use 1 LDS buffer to save LDS memory. + # this can help to generate Kernel that LDS usage originally exceed MaxLDS if using double LDS buffer, + # or help to increase Occupancy. + # 1 means: Force to use 1 LDS Buffer even with PrefetchGlobalRead + # -1 means: generator will use 1 LDS buffer only when LDS exceed MaxLDS + # Use case: + # SIA2: 1LDSBuffer is set to 1 natively + # SIA3: 1LDSBuffer works only when PGR=True + # TODO: optimize scheduling to support more cases. + "1LDSBuffer": [-1, 0, 1], + # Split the unroll summation into multiple sections and combine the sections + # GSU applies only to the unroll summation dimension + # Set to 0 to disable GSU, kernel code will be generated without GSU support + "GlobalSplitU": list(range(0, 1024 + 1)), + # choose how to do GlobalSplitU + # 1: use atomic operation to accumulate on one buffer + # 2: each GSU group write to each own buffer and accumulate by another kernel + # 3: each GSU group write to each own buffer and accumulate by same kernel + "GlobalSplitUAlgorithm": ["SingleBuffer", "MultipleBuffer", "MultipleBufferSingleKernel"], + # don't create a whole copy of the Unroll loop with loads removed - instead + # use buffer limits to suppress global loads and ignore unnecessary ds_reads + "SuppressNoLoadLoop": [False, True], + # For PrefetchGlobalRead=1, create a second copy of the unroll loop with + # the LDS pointer swaps expanded into inline constants for LDS read and write instructions + # This eliminates 4 vector XOR instructions used for pointer swap + "ExpandPointerSwap": [False, True], + # Schedule global reads and global read increments into LocalRead iterations + # Can reduce pressure on local read instruction dispatch queue + # 0=perform global reads at start of instruction loop + # 1=schedule into the local read instruction iterations + "ScheduleGlobalRead": [0, 1], + # Schedule local writes into LocalRead iterations. + # Can reduce pressure on local read instruction dispatch queue + "ScheduleLocalWrite": [0, 1], + # Scheduling algorithm to use for each iteration: + # 0 = minimal/no scheduling. Global Read and increments, followed by local reads, + # followed by local writes, followed by MACs + "ScheduleIterAlg": [0, 1, 2, 3], + # For MatrixInstruction and SIA3, number of GlobalReadInstruction between mfma + # the purpose of this parameter is to control density of global read instruction scheduling + # Scheduling global read back to back can have better memory efficiency + # However, when full of vmem FIFO, it will block other instruction to be issued + # Range from 0.01 to 32 + # 0.1 means 1 GR per 10 mfma + # 5 means 5 GR per 1 mfma + "GlobalReadPerMfma": [i / 100 for i in range(1, 3200)], + # + # For MatrixInstruction and SIA3, number of LocalWriteInstruction between mfma + # the purpose of this parameter is to control density of local write instruction scheduling + # In PGR1, we want to schedule local write more denser, so we can have more + # latency to hide global read + # In PGR2, since LW is followed by GR, every LW has same whole loop latency + # to hide global read. We want to schedule LW less denser, can + # avoid full of vmem FIFO. + # Range from 0.01 to 32 + # 0.1 means 1 LW per 10 mfma + # 5 means 5 LW per 1 mfma + # -1 will derived an optimized value internally + # -2 will derived an optimized value and override LWPM silently (debug only, not recommended) + "LocalWritePerMfma": [i / 100 for i in range(1, 3200)] + [-1], + # Interleave alpha scale calculation with beta loads and address calcs - rather + # than as a separate block of instructions + "InterleaveAlpha": [0, 1], + # Create a copy of NoLoadLoop which interleaves the stores with the final mac + # calculation and may perform other optimizations + # 0 = no interleave + # 1 = interleave one stores after required macs have completed execution + # 2 = interleave two stores after required macs have completed execution + "OptNoLoadLoop": [0, 1, 2], + "BufferLoad": [False, True], + "BufferStore": [False, True], + # Attempt to load directly from global memory into Vgpr. + # Assembly only + "DirectToVgprA": [False, True], + "DirectToVgprB": [False, True], + "DirectToVgprSparseMetadata": [False, True], + # Attempt to load directly from global memory into LDS. + # Assembly only + # Requires BufferLoad, assembler support for lds modifier on buffer + # loads (checked automatically), GlobalVectorWidth=1 (this is hw + # requirement) and A/B must not require any transpose. + # DirectToLds reduces load latency and eliminates the + # G2L registers used to stage data. Also replaces the + # local write offset with an SGPR. + # For an 8x8 TT with PrefetchGlobalRead=1 this can save 33 VGPRs. + # - Requirements for DirectToLds=1: + # GlobalReadVectorWidth = 1/2/4 (GRVW * bpe must be 4 for now) + # TransposeLDS = 1 for TLU=0 case + # DirectToLds support for x1 only for now + "DirectToLds": [False, True], + # Load options: + # (GRO = Global Read Offset) + # BufferLoad=0: + # = Use flat instructions with 64 bit GRO for each load + # + supports sizes up to 2^64 + # - uses many VGPR for addressing + # - uses execmask+compares for edge detection + # - generates extra LDS traffic (could convert flat->global load) + # BufferLoad=1: + # = Use buffer load instructions with 32-bit offset + # + Less VGPRS (32b offset vs 64-bit) needed for addressing + # + Uses hardware buffer limit for edge detection + # - Limited range - the bot-right corner of macro-tile (plus padding=GRVW + # for shift-pointer, if ShiftPtr is required) must be within 2^32. + # ShiftPtrPad = MayShift ? GRWV*BPE : 0 + # For TLU=1: Unroll*StrideA1 + ShiftPtrPad <= 2^32 + # For TLU=0: MT*StrideA1 + ShiftPtrPad <= 2^32 + # These conditions should be checked using Assert - TODO + # = UseSgprForGRO=1: + # + Attempt to use SGPR for Global Read Offsets. + # + Use one VGPR base GRO + many SGPR GRO rather than many VGPR GRO. + # + Each SGPR stores an offset from base GlobalReadOffset+0. + # - Requirements for UseSgprForGRO=1: + # - BufferLoad=1 + # - Use appropriate Assert*ElementMultiple or GRVW=1 to eliminate need for ShifPtr + # (UseSgprForGRO does not support ShiftPtr since ShiftPtr needs to potentially shift GRO) + # = KernelWriterAssembly also supports 64-bit 2D buffer size (see use64bPbcLimit) + # - Requires 4 instructions to move scalar limit and a couple SGPR + # - Enabled by default. If the overhead matters we can add asserts/YAML parm to specialize + # = UseInstOffsetForGRO=1: + # + Attempt to use Instruction offset for Global Read Offsets. + # + This feature avoid updating m0 for subsequent GRO(s) for directToLds feature + # - Requirements for UseInstOffsetForGRO=1: + # - BufferLoad=1 + # - DirectToLds=1 + # converting m0 update from LocalWriteAddrSGpr using is usually win + # -1 attempt to use a heuristic to determine when the tile size will use too many SGPR and fall back to VGPR + "UseInstOffsetForGRO": [-1, 0, 1], + # Converting VGPR GRO into SGPR GRO is usually a win + # However, the mode may exhaust all available SGPR, in particular for large unroll + # -1 attempt to use a heuristic to determine when the tile size will use too many SGPR and fall back to VGPR + "UseSgprForGRO": [-1, 0, 1], + # Use a 64-bit shadow limit register to allow buffers larger than 2^32 bytes + "Use64bShadowLimit": [True, False], + # Assertion properties + # These provide information or assertions that the problem size meets certain requirements + # for sizes or alignments. The kernel generator can use this information to produce + # a kernel which uses those assertions to produce a faster kernel. + # + # If modifying or adding Assertions also change ProblemProperties class in TensileTypes.h + # Kernel generator will assume that the summation size is some multiple of the element size + # and uses this to optimize the kernel. + # This can result in more efficient kernels, but requires runtime checking to ensure the specified + # summation value meets the requirements. + # (Recommended AF1EM value is 8 for half, 4 for single, 2 for double) + # + # Optimizations enabled by AssertSummationElementMultiple>1: + # - If >=2 for half: + # - Tail loop loads can be vectorized 2X to use dword + # - Enables asm kernels on V20 + # - Can use DirectToLds for both unroll and tail loops + # - Tail loop can be unrolled up to InnerUnroll amount if AssertSummationElementMultiple%InnerUnroll==0 + # + # 1 indicates no assertion (since all sizes are multiples of 1) + "AssertSummationElementMultiple": [1, 2, 4, 8, 16, 32, 64, 128], + # Kernel generator will assume that the FreeIndex[0] size is some multiple of the element size + # and uses this to optimize the kernel. + # FreeIndex[0] is usually letter "I" + # (Recommended AF0EM value is 8 for half, 4 for single, 2 for double) + # + # Optimizations enabled by AssertFree0ElementMultiple>1: + # Load optimizations: + # - For TLU=1 matrix, if AF1WM>=GLVW then can enable UseSgprForGRO + # - Reduces registers used for address calculations + # - Removes address shift/unshift code + # - UseSgprForGRO will only be enabled if all matrices meet assertion requirements. + # + # Store Optimizations: + # - Can vectorize stores in edge tiles. Vector width can be up to AF0EM. + # (since C matrix is always coalesced in Free0 index direction and this assertion guarantees the index element multiple) + # + # 1 indicates no assertion (since all sizes are multiples of 1) + "AssertFree0ElementMultiple": [1, 2, 4, 8, 16], + # Kernel generator will assume that the FreeIndex[1] size is some multiple of the element size + # and uses this to optimize the kernel. + # FreeIndex[1] is usually letter "J" + # (Recommended AF1EM value is 8 for half, 4 for single, 2 for double) + # Optimizations enabled by AssertFree1ElementMultiple>1: + # - See above AssertFree0ElementMultiple "Load optimizations" + # 1 indicates no assertion (since all sizes are multiples of 1) + "AssertFree1ElementMultiple": [1, 2, 4, 8, 16], + # Assertions that require arithmetic intensity to be specified value. + # Arithmetic intensity measures the ratio of computation to memory bandwidth required for a problem. + # These predicates can be used to adjust solution selection compute-bound or memory-bound problems. + "AssertAIGreaterThanEqual": -1, + "AssertAILessThanEqual": -1, + # Stagger the start summation position of the tiles. + # Elements from the summation dimension are loaded at offsets rather than all starting at 0. + # StaggerU is the max 'clicks' of StaggerUStride bytes where each wg starts ; see StaggerUMapping + # for how the specific stagger for a given wg is determined. + # + # The tile assignment C are same as with StaggerOffset=0 ; the difference is the + # order that the summation elements are added. + # GRO will wrap back to the row start when the edge is reached. + # + # This can be effective for TLU=0 style matrices where the K dimension is a large power-of-2. + # In this case the start of each row of the tile is separated by an exact power-of-2 + # which causes poor dram, cache, and tlb behavior. V20 has 16 channels each 256 bytes wide. + # StaggerU adjusts the start position in the summation (aka 'U') dimension + # to avoid these conflicts. Both A and B matrix start at the adjusted position. + # If >0 specifies the offset in multiples of the macro-tile "unroll" dim + # - Higher values will spread traffic to more channels but provide less L2 re-use. + # - StaggerU and WorkGroupMapping interact and should be tuned together - + # The WGM controls how tiles are assigned in C matrix, while StaggerU controls where those + # tiles start reading their summation dim parms. + # - StaggerU requires BufferLoad==1 and is silently ignored if BufferLoad==0 + "StaggerU": [0, 2, 4, 8, 16, 32, 64], + # Stride in bytes for each staggeru 'click'. + # 256 is recommended since this is the width of memory channel (on gfx803,gfx900,gf906) - so + # each click will start in a new memory channel and spread traffic among the 16 available channels. + # For example StaggerUStride=256 and StaggerU=8 will use 8 unique starting points + # in summation dimension, each offset by 256-bytes - provided the tensor dims are large + # enough to support this. + # StaggerUStride will be internally increased so it is an integer multiple of DepthU*BpeAB. + # (the implementation requires this - the unroll iteration accesses data in steps of + # DepthU*BPE + "StaggerUStride": [-1, 16, 32, 64, 128, 256, 512, 1024, 2048], + # How the tile assignment (wg0, wg1, wg2) controls the initial StaggerU offset: + # 0: Use wg0 + # 1: Use wg1 + # 2: Use wg2 + # 3: Use wgSerial, wgSerial = wg0 + wg1 * nwg0 + wg2 * (nwg0 * nwg1) + # 4: Debug mode, offset each tile max allowed StaggerU. This just moves hotspot + # to a different bank since all workgroups still start at same point. + "StaggerUMapping": [0, 1, 2, 3, 4], + # GSU Workgroup Coalesced Ordering + # False: {(wg0,wg1,wg2,wgn)|(wg0,wg1,wg2,wgn)|...|(wg0,wg1,wg2,wgn)} + # True: {(wg0,wg0,wg0)|(wg1,wg1,wg1)|(wg2,wg2,wg2)|...|(wgn,wgn,wgn)} + "GlobalSplitUCoalesced": [False, True], + # GSU Workgroup Mapping + # False: wg issued order = {(wg0,wg1,wg2,wgn),(wg0,wg1,wg2,wgn)|...|(wg0,wg1,wg2,wgn)} + # -> workgroups do the summation by tile -> slower GR but faster GW + # True: wg issused oder = {(wg0,wg0,wg0)|(wg1,wg1,wg1)|(wg2,wg2,wg2)|...|(wgn,wgn,wgn)} + # -> workgroups split up the summation -> faster GR but slower GW + "GlobalSplitUWorkGroupMappingRoundRobin": [False, True], + # 0=don't use magic div (source only) + # 1=magic div alg #1. Slightly faster but limited range (if magic number is 2^32) + # 2=magic div alg#2. Slightly slower but handles all unsigned ints up to 2^32 + "MagicDivAlg": [0, 1, 2], + # For Block Mapping type: + # 0 : Use hardware-assigned wg number with no remapping. + # N : WG block width. "Wrap" to a new wg1 "row" assignment after N WGs assigned in that row. + # Tensor C always mapped with first free coord as fastest moving + # (Elements in this dimension are sequential in memory. + # + # For 2D nonbatched Matrix this means index order is I, then J + # For 2D batched Matrix this means index order is I, then J, then K. + # + # Then for 2D case: + # - If drawn in row-major format, I is the width and J is the height. + # - WGM determines dimensions of the box used to assign tiles from C + # - WGM is the height of the box (in the J dimension) + # - Given WGM, the box width (in I dim) is determined by number of CUs + # - The box always moves across matrixC in the fastest-moving "I" dim, then + # wraps to next J. TODO - might be useful to change this? + # + # Examples for 2D matrix: + # WGM=8: on CU64 machine this is a square box + # WGM=1: Short/Fat - this will cover maximum width in I dimension of C. This matches hardware assigned mapping. + # WGM=64: Tall/Skinny - this will cover maximum width in J dimension of C. + # + # Formula for wgSerial: + # wgSerial = wg0 + (wg1 % WorkGroupMapping) * nwg0 + "WorkGroupMapping": list( + range(-1024, 1024 + 1) + ), # change a workgroup's id so that the all the workgroups on the gpu at a time are hitting L2 cache the best + "WorkGroupMappingXCC": [ + 1, + 2, + 4, + 8, + 16, + 32, + ], # change a workgroup's id so that contiguous workgroup can map on same XCC + # -1 : WorkGroupMappingXCCGroup will be set to CU_count at runtime. Please ensure that (CU_count % WGMXCC == 0). + "WorkGroupMappingXCCGroup": list( + range(-1, 1024) + ), # change a workgroup's id so that contiguous workgroup can map on same XCC, remap workgroup in a group of WGMXCCG. + "MaxOccupancy": list( + range(1, 40 + 1) + ), # wg / CU; if cache thrashing is hurting performance, this allocates extra lds to artificially limit occupancy + "WorkGroup": validWorkGroups, # ( wg0 x wg1 x LocalSplitU ) dimensions of the workgroup which will operate on a tile and share lds + # ThreadTile: ( tt0 x tt1 ) dimensions of the C tile that each thread works on, + # TT=4 and VW=4 means a thread will work on a tight 4x4 tile of C, where VW=1 means the tile will work on 16 spread out values + # Generally, the VW determines the consecutive a WI will work on, then it will skip ahead SG0*VW elements to get to the next row of VGPR inputs + "ThreadTile": validThreadTiles, + "MacroTile": validMacroTiles, # MT0 = wg0*tt0, MT1 = wg1*tt1 + "WavefrontSize": [32, 64], + # MatrixInstruction: (M x N x K x B) + # XDLOPS tile definition, only valid for gfx908, gfx90a + # MxNxKxB specifies matrix instruction variants + # MxNxB determines the shape of the C tile each instruction worked on + # K determines the unroll depth + # If empty, do not use these instructions + # + # Alternative format: (M x N x K x B x MIBlockM x WaveTileM x WaveTileN x WaveM x WaveN) + # (Note: MxN means M-by-N in the following comments) + # MIBlockM determines how many blocks along M dimension for multi-block MI variants. Concrete examples: + # - MI 16x16x1x4 (4-block variant) with MIBlockM=4 -> (16x16)*(4x1)=64x16 tile per instruction executed + # - MI 32x32x1x2 (2-block variant) with MIBlockM=1 -> (32x32)*(1x2)=32x64 tile per instruction executed + # WaveTileM/N are dimensions of the C tile each wave works on, and is close to the concept of ThreadTile in classic VALU kernels + # - WT 4x1 -> each wave executes 4x1 matrix instructions on the C tile of total area (4*MITileM)x(1*MITileN) + # WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads + # - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2 + # Putting it all together: + # - [32, 32, 1, 2, 1, 4, 1, 2, 2] + # ^^^^^^^^^^^^ ^ ^^^^ ^^^^ + # MatrixInst BlkM WT Wave + # - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile + # Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used + # NOTE: MatrixInstruction is no longer validated through this structure, but is instead validated via the + # ``TensileLogic`` program. + "MatrixInstruction": -1, + # StoreRemap: Optimize MatrixInstruction store patterns to enhance performance. + # MI output data between each threads are along N dims. + # But global memory is along M dim continuous. + # That mean global write between each threads are not continuous. + # Therefore, store performance for MI instruction is poor. + # How StoreRemap works in final store stage: + # 1. Put all thread output data into LDS. + # 2. All thread read data from LDS along M dims. + # (match global Memory continuous direction) + # 3. All thread write out data into global memory. + # 0: Disable StoreRemap (default) + # 1~8: Enable StoreRemap and set the global write vector width + # Suggest optimum value: fp32 = [2,4], fp16 or bf16 = [4,8] (dwordx2 and dowrdx4) + # -1: Use dwordx2 if support SRVW, or set SRVW to 0 + "StoreRemapVectorWidth": [-1, 0, 1, 2, 4, 8], + # SourceSwap: Optimizes MatrixInstruction store pattern by swapping mfma input order. + "SourceSwap": [False, True], + # Following parameters are designed for store scheduling. + # (store stands for load from C (with beta) and store to C/D) + # + # we want to hide store behind unroll loop + # 1. if we can launch 2 WorkGroups per CU (occupancy >= 2, large M/N) + # 2. if there are remaining global memory bandwidth in unroll loop (compute bound kernel) + # + # we can hide store behind the other WG's loop by lowering priority of store + # priority of loop is the same as priority of store + # WG0: ???????????????\__ + # |<-- loop --->|<-- store -->|end + # + # WG1: ___________________________/????????????\__ + # |<--------- loop ------------------->|<-- store -->|end + # + # priority of loop is higher than priority of store + # WG0: ???????\____________________ + # |<-- loop --->|<------ store ----->|end + # + # WG1: _____________/?????\__________________ + # |<------- loop -------->|<----- store ---->|end + "StorePriorityOpt": [False, True], + # + # If we issue store in short period of time, kernel will become from compute bound to memory bound + # 0 means issue instructions as many as possible if VGPR available + "NumElementsPerBatchStore": list(range(-1, 256)), + # + # add sync after per batch store in order to store contiguous elements + # add sleep after per batch store in order to distribute store over whole loops + # NOTE: this parameter is highly depends on size_k + # 0 means no sync and sleep + "StoreSyncOpt": list(range(0, 256)), + # + # There are index or address calculation between global instructions. + # issue global instruction b2b has better performance + "GroupLoadStore": [False, True], + # In order to remove the copying from Acc vgpr to Arch vgpr, only use Arch vgprs for v_mfma_xxx. + # Only support for kernel whose totalVgpr counts less than 256 and gcn that has control bit ACC_CD. + "MIArchVgpr": [False, True], + # StreamK (SK) kernels divide work evenly among CUs by splitting along MT and K dimensions. + # Total work units are calculated as (#MTs x #LoopIters) and divided among workgroups. + # In most cases each workgroup will calculate a partial tile that are accumulated in a fixup step in the same kernel + # 0 : Standard data-parallel kernel + # 1 : Basic StreamK + # 2 : Two-Tile StreamK (each WG completes an even number of sk iterations, followed by an even number of dp tiles) + # 3 : Two-Tile StreamK with DP before SK tiles + # StreamK kernels can adjust the number of CUs being used. + # Using fewer sometimes increases overall throughput by allowing other kernels to run in parallel. + # StreamK grid is controlled by setting these enviornment variables: + # TENSILE_STREAMK_FIXED_GRID lets you override the default grid size with a specific number + # 0 = override disabled (default) + # TENSILE_STREAMK_FULL_TILES sets the number of full tiles to be included in stream-k work + # -1 = use prediction model for best performance (not yet implemented) + # 0 = only remainder tiles run in stream-k + # 1+ = remainder + 1 (or more) full grids of tiles run in stream-k (default=1) + # TENSILE_STREAMK_DYNAMIC_GRID selects dynamic grid mode, which automatically limits the number of CUs used: + # 0 = Off, always use all CUs. + # 1 = Only reduce CUs for small problems to number of output tiles when num_tiles < CU count. + # 2 = Also reduce CUs used for large sizes to improve data-parallel portion and reduce power. + # 3 = Analytically predict the best grid-size by weighing the cost of the fix-up step and the cost of processing MACs (default). + # Note: dynamic grid coefficients currently apply to gfx942 variants + # TENSILE_STREAMK_MAX_CUS allows the user to manually set maximum number of CUs used, which could free up some CUs for + # other operations to run in parallel with gemm. + # TENSILE_STREAMK_GRID_MULTIPLIER lets you set how many workgroups are created per CU being used. + # 1 = 1 WG per CU (default), for example. 2 will launch WGs = 2 x CU count. + # The priority of these environment variables is defined as follows: + # TENSILE_STREAMK_FIXED_GRID > TENSILE_STREAMK_DYNAMIC_GRID > TENSILE_STREAMK_MAX_CUS > TENSILE_STREAMK_GRID_MULTIPLIER + "StreamK": [0, 1, 2, 3], + # Determines if StreamK kernel uses atomics + # 0: uses workspace to store partial tiles, accumulate in deterministic fix-up step + # 1: uses atomics to accumulate partial tiles + "StreamKAtomic": [0, 1], + # Enables XCC-based remapping of workgroups, set the value to the number of XCCs + # for the device/configuration being used + # 0: uses default workgroup assignment + # 2+: remaps workgroups to be contiguous within an XCC for a given number of XCCs + "StreamKXCCMapping": [0] + list(range(2, 9)), + # Debug settings for stream-k kernels to disable parts of the kernel + # Bit 0: Don't generate fixup code + # Bit 1: Don't generate write to partials code + # Both parts can be disabled together + # 0 = Debug mode off, generate full kernel + # 1 = No fixup + # 2 = No partials + # 3 = Nofixup and no partials + "DebugStreamK": [0, 1, 2, 3], + # Controls desired width (#elements) for loads from global memory -> LDS. + # and eliminates the pointer unshift logic + # -1 : Set GlobalReadVectorWidth = VectorWidth + # NOTE: for input bpe=32, max GRVW is 4 (to fit dwordx4) (FP32), min GRVW is 1 (dword) + # bpe=16, max GRVW is 8 (to fit dwordx4) (FP16), min GRVW is 2 (dword) + # bpe=8, max GRVW is 16 (to fit dwordx4) (INT8), min GRVW is 4 (dword) + "GlobalReadVectorWidthA": [-2, -1, 1, 2, 3, 4, 6, 8, 16], + "GlobalReadVectorWidthB": [-2, -1, 1, 2, 3, 4, 6, 8, 16], + # Controls desired width (#elements) for loads from LDS -> VGPR. + # -1 : Set LocalReadVectorWidth = VectorWidth + # 1 cannot be used for half type. + # used in combination with TransposeLDS=True + # in TransposeLDS=1 case, use wider load to fetch elements in summation dimension from LDS + # helps optimizing instruction scheduling between MFMA and nonMFMA instructions + # NOTE: for input bpe=32, max LRVW is 4 (to fit ds_read_b128) (FP32) + # bpe=16, max LRVW is 8 (to fit ds_read_b128) (FP16) + # bpe=8, max LRVW is 16 (to fit ds_read_b128) (INT8) + "LocalReadVectorWidth": [-1, 1, 2, 4, 8, 16], + # threads should read/write/operate on this many contiguous elements from the C matrix. + # If VW=4 then thread0 will process 4 consec C elements, then thread1 next 4, etc. + # If the ThreadTile is > VectorWidth then thread0 will next operate on the 4 elements in C at (4*NumThreads) + # Typically the load vector width and store vector width are directly related to the VW. + # The global load width is closely related to the width of local stores so + # GlobalReadVectorWidth also controls local write width. + # Local read width also matches since VectorWidth consec elements must be read + # Typically matching 16 bytes is good choice since the stores will be optimally coalesced with 16 bytes/WI. + # Using a VW too large which results in >16bytes/thread isn't supported + # For MFMA non SourceSwap: this parameter didn't take effect + # -1 means set vw to largest localReadWidth according to MIWaveTile + "VectorWidthA": [-1, 1, 2, 3, 4, 6, 8], + "VectorWidthB": [-1, 1, 2, 3, 4, 6, 8], + # If 0, store 1 element per instruction. + # If 1, store vector-width elements per instruction. + # if -1, store vector-wide elements per instruction unless PBD would not generate a valid kernel + "VectorStore": [-1, 0, 1], + # Controls desired width (#elements) for stores from reg to global memory. + # When MatrixInstruciton == None, derived parameter gwvw takes precedence. + # -1 : Set StoreVectorWidth = VectorWidth + "StoreVectorWidth": [-1, 1, 2, 3, 4, 6, 8], + # when loading all the data from global into lds requires multiple load instructions, these parameters govern which + # loads will pull which rectangle of data from global into lds + # NLC=1 means one load along the coalesced dimension, which results in the most coalescing possible + # NLC=-1 looks for the largest number of reads along the coalesced dimension which results in the least ammount of coalescing; + # however in this case the stride between one load and another is a static value, therefore buffer loads only need one set of registers + # whereas the =1 case has a stride which is a multiple of a kernel argument and therefore needs one address per load in the perpendicular dimension + "NumLoadsCoalescedA": list(range(-1, 64 + 1)), + "NumLoadsCoalescedB": list(range(-1, 64 + 1)), + # DepthU, LocalSplitU (which is the 3rd number in WorkGroup), and LoopUnroll are closely related + # LoopUnroll=4 means there are 4 subiterations within the loop, 4 actual iterations written in the code. + # LocalSplit=2 means the workgroup is split up into 2 subgroups, and each subgroup is doing different parts of the summation. + # subgroup0 does k=0-3, 8-11... and subgroup1 does k=4-7, 12-15... + # So, each iteration through the summation loop, which has 4 actual subiterations, does 8 summation iterations, because each subgroup did 4; + # and when data is read from global memory the threads read 8 elements along the summation dimension. + # DepthU = LoopUnroll * LocalSplitU = 4*2 in this case + # it made more sense for the user to directly control LocalSplitU and DepthU, then derrive afterwards LoopUnroll=DepthU/LocalSplitU + # -1 : Only allow GLVW=1 + # -2 : Only allow max(GLVWA,GLVWB) < VW ? + # -3 : Only allow min(GLVWA,GLVWB) < VW ? + "DepthU": depthUs, + # integer amount of padding to put into LDS, in 2016 this didn't seem to help performance, profilers were showing that channel conflicts weren't really hurting + # performance so this has been deprecated and probably doesn't work + # -1 means use same padding as the VectorWidth if TLU=0 else 0. (Padding only helps when transpose is required) + # With MatrixInstruciton: -1 means max(GRVW,MIInput) if TLU=0 + "LdsPadA": [-1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64], + "LdsPadB": [-1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64], + "LdsPadMetadata": [-1, 0, 1, 2, 3, 4, 8], + # Padding boundary for LDS. defines block-size for pad insertion. for every 'LdsBlockSizePerPad' bytes, LDS padding (pad value from LdsPad parameter) + # is added (readOffset aware of the pad and adjusts offset value based on this parameter value). + # Only support LdsBlockSizePerPad >= unrollDepth * BPE + # 0 means disable LdsBlockSizePerPad + "LdsBlockSizePerPadA": [-1, 0, 64, 128, 256, 512, 1024, 2048], + "LdsBlockSizePerPadB": [-1, 0, 64, 128, 256, 512, 1024, 2048], + "LdsBlockSizePerPadMetadata": [-1, 0, 64, 128, 256, 512, 1024, 2048], + # Transpose LDS format. Local store in coalesced dimension , same as optimized global fetch dimension . applicable only in TLU=0 case for miSIMD(s) + # -1 : keep LDS layout same as global fetch dimension for both A and B + # set TLDS = 1 for NN,TN,TT + # set TLDS = 0 for NT + # 0 : coalesced dimension of lds is tile dimension + # 1 : keep LDS layout same as global fetch dimension for both A and B for NN,TN,TT, but NT would be rejected + # 2 : coalesced dimension of lds is unroll dimension for both A and B + "TransposeLDS": [-1, 1, 0, 2], + # add gls or slc after global memory read/writes to change caching, not caching the writes is promising and improved performance a tiny bit + # 0: none, 1: glc, 2: slc, 3: glc slc + # For gfx942, sets sc0/sc1/nt bits + # 0: none, 1: sc0, 2: sc1, 3: sc0 sc1, 4: nt, 5: nt sc0, 6: nt sc1, 7: nt sc0 sc1 + "NonTemporalE": list(range(0, 8)), + "NonTemporalD": list(range(0, 8)), + "NonTemporalC": list(range(0, 8)), + "NonTemporalA": list(range(0, 8)), + "NonTemporalB": list(range(0, 8)), + "NonTemporalWS": list(range(0, 8)), + "NonTemporalMetadata": list(range(0, 8)), + "NonTemporal": list(range(-1, 8)), + # Group together unroll iterations inside the unroll loop. + # For example, InnerUnroll=2 will fetch LDS for two unroll iterations + "InnerUnroll": [1, 2, 4, 8, 16, 32, 64], + # Enable CP preload kernel arguments feature + # It can reduce time of loading kernel arguments by s_load. + # It needs new complier and vbios to support this feature. + "PreloadKernArgs": [False, True], + # Kernels should be written in assembly or source + # if assembly, ISA will determine architecture + # if source, Runtime will determine language + # later on, we'll relax this to inner kernel languages and outer kernel languages, such as inline asm embedded in ocl or in llvm + "KernelLanguage": ["Assembly"], + # We set validParams["ISA"] in multiple places + "ISA": validISA, # arch for assembly kernels + # Name of the custom kernel located at `CUSTOM_KERNEL_PATH`. + # a custom kernel is a user written assembly kernel with its associated configuration parameters included in a custom.config section + # inside the yaml block between the --- and ... markers. These parameters are only used for information purposes, not kernel generation. + # Ex: + # custom.config: + # ProblemType: + # OperationType: GEMM + # etc... + # ThreadTile: [8, 8] + # etc... + # + # Custom kernels can be included in a BenchmarkProblemSizeGroup by having their name (without file extension) listed under the "CustomKernels" + # category alongside InitialSolutionParameters, BenchmarkCommonParameters, etc... + "CustomKernelName": -1, + # Will allow a kernel to be accepted even when checks determine it's not viable. + # Intended for use with custom kernels which have confirmed to be correct + "NoReject": [False, True], + # Debug use only. + "ActivationFused": [False, True], + # True- function call + # False- inline + "ActivationFuncCall": [False, True], + # Alternative implementation for activation function + # Currently only supports GSU == 1 + "ActivationAlt": [False, True], + # Do workgroup reduction. Currently for DBias + "WorkGroupReduction": [False], + # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B + "Sparse": [0, 1, 2], + # in mix mode F8 need to convert to F16, do this before(0) ds or after(1) ds + "ConvertAfterDS": [False, True], + # Force disable shadow init to release more sgpr in preloop + "ForceDisableShadowInit": [False, True], +} + +newMIValidParameters = { + "EnableF32XdlMathOp": [False, True], + 'EnableMatrixInstruction': [False, True], + 'ISA': -1, + 'MFMA_BF16_1K': [False, True], + 'MIBlock': -1, + 'MIInputPerThread': -1, + 'MIInputPerThreadA': -1, + 'MIInputPerThreadB': -1, + 'MIInputPerThreadMetadata': -1, + 'MIWaveGroup': -1, + 'MIWaveTile': -1, + 'MatrixInstM': -1, + 'MatrixInstN': -1, + 'MatrixInstK': -1, + 'MatrixInstB': -1, + 'MatrixInstBM': -1, + 'MatrixInstBN': -1, + 'MatrixInstruction': -1, + 'Sparse': -1, + 'ThreadTile': -1, + 'WavefrontSize': -1, + 'WorkGroup': -1, +} + + +def checkParametersAreValid(param, validParams): + """Ensures paramaters in params exist and have valid values as specified by validParames""" + (name, values) = param + if name == "ProblemSizes": + return + elif name == "InternalSupportParams": + return + + if name not in validParams: + raise Exception( + "Invalid parameter name: {}\nValid parameters are {}.".format( + name, sorted(validParameters.keys()) + ) + ) + + for value in values: + if validParams[name] != -1 and value not in validParams[name]: + msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}." + msgExt = ( + " (only first 32 combos printed)\nRefer to Common.py for more info" + if len(validParams[name]) > 32 + else "" + ) + raise Exception(msgBase.format(name, value, name, validParams[name][:32], msgExt)) diff --git a/tensilelite/Tensile/Common/__init__.py b/tensilelite/Tensile/Common/__init__.py index 2addea2cd1..b49aa2ee79 100644 --- a/tensilelite/Tensile/Common/__init__.py +++ b/tensilelite/Tensile/Common/__init__.py @@ -1,10 +1,14 @@ from .Architectures import * from .Capabilities import * from .Constants import * - -# Dunder variables are not exported via `*` from .GlobalParameters import * +# Dunder variables are not exported via `*` from .GlobalParameters import __version__ +# from .Naming import * from .Parallel import * from .Types import * from .Utilities import * + +# NOTE: Do not export valid parameters automatically to save memory +# it must be explicitly imported where needed +# from .ValidParameters import * diff --git a/tensilelite/Tensile/Components/ComputeStoreVgprs.py b/tensilelite/Tensile/Components/ComputeStoreVgprs.py index 5209c34660..cda15cc55c 100644 --- a/tensilelite/Tensile/Components/ComputeStoreVgprs.py +++ b/tensilelite/Tensile/Components/ComputeStoreVgprs.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -27,7 +27,7 @@ vectorStaticRemainder, RegisterPoolResource, vgpr, sgpr, log2, \ vectorStaticDivideAndRemainder from ..Component import ComputeStoreVgprs -from ..Common import DataDirection +from ..Common import DataDirection, printExit, printWarning class ComputeStoreVgprsVALU(ComputeStoreVgprs): kernel = {"EnableMatrixInstruction": False, diff --git a/tensilelite/Tensile/Components/GlobalWriteBatch.py b/tensilelite/Tensile/Components/GlobalWriteBatch.py index d0d1204312..790d2c78ef 100644 --- a/tensilelite/Tensile/Components/GlobalWriteBatch.py +++ b/tensilelite/Tensile/Components/GlobalWriteBatch.py @@ -20,7 +20,7 @@ # CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ################################################################################ -from ..Common import globalParameters, DataDirection, SemanticVersion +from ..Common import DataDirection, SemanticVersion from ..Component import GlobalWriteComponents from ..SolutionStructs import Solution from ..Activation import ActivationModule, ActivationType diff --git a/tensilelite/Tensile/Components/PackData.py b/tensilelite/Tensile/Components/PackData.py index fe3a833f9b..e52528a010 100644 --- a/tensilelite/Tensile/Components/PackData.py +++ b/tensilelite/Tensile/Components/PackData.py @@ -29,7 +29,6 @@ vgpr, sgpr, DataType, TensileInstructions, VAndB32, \ VMovB32, VLShiftLeftB32 from ..Component import PackData -from ..Common import globalParameters def formatting(idx, inputPrefix, prefixOffset): if inputPrefix: diff --git a/tensilelite/Tensile/Components/SIA.py b/tensilelite/Tensile/Components/SIA.py index 50bb30745f..b41cd7dd57 100644 --- a/tensilelite/Tensile/Components/SIA.py +++ b/tensilelite/Tensile/Components/SIA.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tensilelite/Tensile/Components/Signature.py b/tensilelite/Tensile/Components/Signature.py index 124089842f..7b7ce0aa40 100644 --- a/tensilelite/Tensile/Components/Signature.py +++ b/tensilelite/Tensile/Components/Signature.py @@ -23,7 +23,7 @@ ################################################################################ from ..Component import Signature -from ..Common import globalParameters, DataDirection +from ..Common import DataDirection from ..TensileInstructions import SignatureBase from ..TensileInstructions import SignatureValueKind as SVK from ..Activation import ActivationType @@ -118,7 +118,7 @@ def __call__(self, writer) -> SignatureBase: kernArgReg -= 2 # strides kernArgReg += kernel["ProblemType"]["NumIndicesSummation"] kernArgReg += kernel["ProblemType"]["NumIndicesC"] - if globalParameters["DebugKernel"]: + if writer.debugConfig.debugKernel: kernArgReg += writer.states.rpga # debug buffer # kernArgBytes = kernArgReg * 4 # bytes/reg @@ -159,7 +159,7 @@ def __call__(self, writer) -> SignatureBase: signature.addArg( "SizesSum%u"%i, SVK.SIG_VALUE, "u32") userArgumentsInfo.gemmArgumentSize += 4 - if globalParameters["DebugKernel"]: + if writer.debugConfig.debugKernel: signature.addArg("AddressDbg", SVK.SIG_GLOBALBUFFER, "struct", "generic") signature.addArg( "D", SVK.SIG_GLOBALBUFFER, dstValueType, "generic") signature.addArg( "C", SVK.SIG_GLOBALBUFFER, dstValueType, "generic") diff --git a/tensilelite/Tensile/Components/StreamK.py b/tensilelite/Tensile/Components/StreamK.py index 42ecb1a580..6b45f4b0d2 100644 --- a/tensilelite/Tensile/Components/StreamK.py +++ b/tensilelite/Tensile/Components/StreamK.py @@ -1536,7 +1536,7 @@ def preLoop(self, writer, kernel): def graWorkGroup(self, writer, kernel, tPA, tPB): module = Module("StreamK Off graWorkGroup") - if writer.states.archCaps["WrokGroupIdFromTTM"]: + if writer.states.archCaps["WorkGroupIdFromTTM"]: module.add(SMovB32(dst=sgpr("WorkGroup0"), src="ttmp9", comment="workaround")) module.add(SAndB32(dst=sgpr("WorkGroup1"), src0=hex(0xFFFF), src1="ttmp7", comment="workaround")) module.add(SLShiftRightB32(dst=sgpr("WorkGroup2"), shiftHex=hex(0x10), src="ttmp7")) diff --git a/tensilelite/Tensile/Contractions.py b/tensilelite/Tensile/Contractions.py index 73706d5c92..f6b41c1234 100644 --- a/tensilelite/Tensile/Contractions.py +++ b/tensilelite/Tensile/Contractions.py @@ -22,14 +22,19 @@ # ################################################################################ +from typing import Dict + from .Activation import ActivationType from .TensileInstructions import DataType from . import Hardware from . import Properties -from .SolutionStructs import getBiasDataTypeListDefault -from .SolutionStructs import Solution as OriginalSolution -from .Common import gfxToIsa, internalParameters, globalParameters, state, state_key_ordering +from Tensile.SolutionStructs import Solution as OriginalSolution +from .Common import gfxToIsa, internalParameters, state, state_key_ordering, \ + IsaInfo, DepthUConfig +from Tensile.SolutionStructs.Problem import getBiasDataTypeListDefault +from Tensile.Toolchain.Component import Assembler +MIN_K_FOR_GSU = 32 @state_key_ordering class FreeIndex: StateKeys = ['isA', 'i', 'c', 'd'] @@ -511,7 +516,7 @@ def CompoundPredicates(cls, state, problemType): rv += [cls('BufferStoreOffsetLimitCheck', value=state['MacroTile1'])] if '_GlobalAccumulation' in state and state['_GlobalAccumulation'] != None and not state["StreamK"]: - value = globalParameters['MinKForGSU'] + value = MIN_K_FOR_GSU rv += [cls('GlobalSplitUCheckMinK', value=[value, state["GlobalSplitU"]])] if ('WorkGroupMappingXCC' in state) and ('WorkGroupMappingXCCGroup' in state): @@ -656,14 +661,43 @@ class Solution: HiddenKeys = ['originalSolution'] @classmethod - def FromSolutionStruct(cls, solution, cxxCompiler: str): - return cls.FromOriginalState(solution._state, cxxCompiler, solution.srcName) + def FromSolutionStruct( + cls, + solution, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + assembler: Assembler, + isaInfoMap: Dict[str, IsaInfo] + ): + return cls.FromOriginalState( + solution._state, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap, + solution.srcName + ) @classmethod - def FromOriginalState(cls, d, cxxCompiler, srcName = "", deviceInfo=None): + def FromOriginalState( + cls, + d, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + #mink + assembler, + isaInfoMap, + srcName = "", + deviceInfo=None + ): rv = cls() - if 'SolutionNameMin' in d: rv.name = d['SolutionNameMin'] @@ -700,14 +734,21 @@ def FromOriginalState(cls, d, cxxCompiler, srcName = "", deviceInfo=None): if 'ISA' not in d: if d['KernelLanguage'] == 'Assembly': d['ISA'] = gfxToIsa(deviceInfo[1]) - else: - d['ISA'] = [0,0,0] if 'CUCount' not in d: d['CUCount'] = None rv.hardwarePredicate = Hardware.HardwarePredicate.FromHardware(d['ISA'], d['CUCount']) - rv.originalSolution = OriginalSolution(d, cxxCompiler, srcName) + rv.originalSolution = OriginalSolution( + d, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap, + srcName + ) rv.srcName = srcName return rv diff --git a/tensilelite/Tensile/CustomKernels.py b/tensilelite/Tensile/CustomKernels.py index 12a8214eb9..9ee5afb60e 100644 --- a/tensilelite/Tensile/CustomKernels.py +++ b/tensilelite/Tensile/CustomKernels.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -23,7 +23,7 @@ ################################################################################ from . import CUSTOM_KERNEL_PATH -from .Common import checkParametersAreValid, validParameters +from .Common.ValidParameters import checkParametersAreValid, validParameters, newMIValidParameters import yaml @@ -65,20 +65,44 @@ def readCustomKernelConfig(name, directory=CUSTOM_KERNEL_PATH): except yaml.scanner.ScannerError as e: raise RuntimeError("Failed to read configuration for custom kernel: {0}\nDetails:\n{1}".format(name, e)) -def getCustomKernelConfig(kernelName, internalSupportParams, directory=CUSTOM_KERNEL_PATH): +def getCustomKernelConfig( + kernelName: str, internalSupportParams: dict, directory: str = CUSTOM_KERNEL_PATH +) -> dict: + """ + Retrieves and validates the configuration for a custom kernel. + + Args: + kernelName: The name of the custom kernel. + internalSupportParams: A dictionary of internal support parameters to be merged with the kernel configuration. + directory: The directory where custom kernel files are located. Defaults to CUSTOM_KERNEL_PATH. + + Returns: + dict: The validated configuration dictionary for the custom kernel. + + Raises: + RuntimeError: If the custom kernel configuration is missing required fields or if there is an error reading the configuration. + """ kernelConfig = readCustomKernelConfig(kernelName, directory) if "InternalSupportParams" not in kernelConfig: - raise RuntimeError("Custom kernel %s config must have KernArgsVersion"%kernelName) - else: - # CustomKernelConfig must have signature version - if "KernArgsVersion" not in kernelConfig["InternalSupportParams"]: - raise RuntimeError("Custom kernel %s config must have KernArgsVersion"%kernelName) - for key in internalSupportParams: - if key not in kernelConfig["InternalSupportParams"]: - kernelConfig["InternalSupportParams"][key] = internalSupportParams[key] + raise RuntimeError(f"Custom kernel {kernelName} config must have 'InternalSupportParams'") + + if "KernArgsVersion" not in kernelConfig["InternalSupportParams"]: + raise RuntimeError(f"Custom kernel {kernelName} config must have 'KernArgsVersion'") + + kernelIsp = kernelConfig["InternalSupportParams"] + for key in internalSupportParams: + if key not in kernelIsp: + kernelIsp[key] = internalSupportParams[key] + + # validParams = validParameters.update(newMIValidParameters) + # if not validParams: + # raise RuntimeError(f"Valid parameters not loaded: {validParameters}\n{newMIValidParameters}\n{validParams}") + validParameters.update(newMIValidParameters) + for k, v in kernelConfig.items(): if k != "ProblemType": checkParametersAreValid((k, [v]), validParameters) + kernelConfig["KernelLanguage"] = "Assembly" kernelConfig["CustomKernelName"] = kernelName diff --git a/tensilelite/Tensile/CustomKernels/CustomGSUs_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_MT128x16x128_MI16x16x1_44_Freesize_gfx942.s b/tensilelite/Tensile/CustomKernels/CustomGSUs_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_MT128x16x128_MI16x16x1_44_Freesize_gfx942.s index b89fc28ae9..77f838e984 100644 --- a/tensilelite/Tensile/CustomKernels/CustomGSUs_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_MT128x16x128_MI16x16x1_44_Freesize_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/CustomGSUs_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_MT128x16x128_MI16x16x1_44_Freesize_gfx942.s @@ -63,7 +63,27 @@ custom.config: Activation: True UseScaleAlphaVec: 1 SupportUserArgs: False - MatrixInstruction: [16, 16, 16, 1, 1, 2,1, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [2, 1] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 DepthU: 128 StaggerU: 4 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname0_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname0_gfx942.s index 3674c844c2..0c70be040e 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname0_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname0_gfx942.s @@ -59,7 +59,27 @@ custom.config: UseBias: 1 Activation: True UseScaleAlphaVec: 1 - MatrixInstruction: [16, 16, 16, 1, 1, 8,1, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [8, 1] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 DepthU: 32 StaggerU: 4 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname1_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname1_gfx942.s index 6e87f5aca6..4ac5825587 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname1_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_AS_SAB_SAV_shortname1_gfx942.s @@ -59,7 +59,27 @@ custom.config: UseBias: 1 Activation: True UseScaleAlphaVec: 1 - MatrixInstruction: [16, 16, 16, 1, 1, 2,2, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [2, 2] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 DepthU: 64 StaggerU: 4 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s index 0c83d6a61c..f4406dfa26 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s index d0f986a3f7..7ef9df1b0c 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s index cd19ced94f..ab4c16ff21 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s index 8491de54b8..bdff04b446 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s index 6271f36e14..981f775192 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s index 297a1458f4..fed7525a1e 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s index bc94ba39f8..cd26c62fe1 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s index 3ebbdfa083..a633806eaa 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,10, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 10] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s index 1f0994c757..c075bcc0a4 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,11, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 11] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s index 0cf92d8ea3..79afec6cf8 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s index 24ab9a9c50..96bf1a55fd 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s index 66541fa0dc..8df5324deb 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s index df3b5ef4b6..e5d308d026 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s index c59cd4b3c7..b56e55e18e 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s index ec0b7fb75a..b3bdc5ebf0 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s index 0931e6a7c7..fcc84176f1 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s index 7321119941..c82e981471 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s index 344361d4cf..28a3fd19cd 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s index e007147706..ceac6d116c 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s index 197890e178..43bcd13213 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s index cc6f743af0..af61cb1e79 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HHS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s index cba2a8b539..98997d7b19 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname0_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s index 94156abc40..f0fc41a8e6 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname10_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s index 1016ce74a1..ea0301810d 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname11_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s index 3c388c6231..86a3300594 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname12_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s index 6066506b6f..be0bbe9fe4 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname13_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s index 28cdb3dd46..63ac8fe345 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname14_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s index 92e9a4d8ac..bccdbc3383 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname15_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s index 1f3443996c..ae7432f8b2 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname16_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,10, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 10] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s index e6c2460ae4..90b1228119 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname17_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,11, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 11] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s index 513376dcac..3d4e7688a1 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname18_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s index 407af1a4dc..7814e350a9 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname19_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s index 6414e7c8f4..ec8bebf7f6 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname1_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s index 916cd5d59e..d51831eadb 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname20_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s index 1c5e1faa88..82273f6ab3 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname2_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s index 5b95c072b8..1a44086769 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname3_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s index e179e481ba..d70720e24f 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname4_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s index ed247a1410..427eaec8f9 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname5_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s index fd84446701..46e3e1c1ab 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname6_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s index ca184cda90..12a72a3edf 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname7_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s index 4e55195289..43b7698221 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname8_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s index 4943104af3..16c2974b35 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_F8NH_HSS_BH_Bias_GG_AS_SAB_SAV_UserArgs_shortname9_gfx942.s @@ -64,7 +64,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s index 8e2409bcda..a4d5590ff4 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s index 2f9ee080b7..e1aae690af 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s index 282cb330d7..eed0581627 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s index 9db4a1cbed..72ca30efb3 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,20, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 20] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s index 9786a5b756..08506c6223 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,20, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 20] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s index 580fd6577a..ffec6819fd 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,20, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 20] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s index da2df8e691..5c5253cde0 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,11, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 11] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s index 64734b241a..2be6c86c66 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,11, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 11] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s index 73ce86d687..013050f3fb 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s index 69360d2ddf..df228c0923 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s index 326ac4eeb3..b3c9839640 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s index 2e22f24983..651040d735 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s index 5da71a8c30..d4c3e21c64 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s index 43b9c3244e..8ba8ce2d80 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s index 7aef4885fc..d6b0adf06c 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s index 85baa5575f..d0e79e9909 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s index f054ea8a09..da6505fd69 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s index 6e40f868e7..a198f6a5af 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s index 4eee79eaf3..0956cb45cc 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HHS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s index ba6a788e2e..7a2d3efadd 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname0_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s index 7f4cc981a8..29e5245b9e 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname10_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s index 85af1877f6..2b8e64eabc 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname11_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s index 052ebf3fbd..c9b2b475bb 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname12_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,20, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 20] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s index afe4d304cd..1acd754b04 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname13_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,20, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 20] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s index 1dbfd0b224..31f955b65c 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname14_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,20, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 20] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s index 7032dcd8f5..a7ef387537 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname15_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,11, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 11] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s index 166d0e4c9d..2ebc5b41c0 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname16_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,11, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 11] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s index fb2216b86d..32260f7145 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname17_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s index ba9e5083a6..64035df006 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname18_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 8,12, 2,2] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [2, 2] + MIWaveTile: [8, 12] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [32, 8, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s index 259bc85cea..dd130c51c0 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname1_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s index 23b5ef82bd..82edeb7b7d 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname2_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s index 337bbbf56a..6a37aecff5 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname3_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,9, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 9] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 0 ScheduleIterAlg: 3 DepthU: 32 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s index 421c90dc87..5cddea6d14 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname4_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s index df6b7e09ec..f6ae123378 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname5_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s index bd6113c409..26461bb12c 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname6_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s index d763c33edb..db27d906b9 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname7_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,14, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 14] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s index eba66bad78..b1d599fc72 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname8_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s index 878991f11d..52d55ad6b8 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Ailk_Bljk_HSS_BH_Bias_GG_AS_SAV_UserArgs_shortname9_gfx942.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: True SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4,16, 4,1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s index ab17bdd17a..cbbd711b93 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_DTVA.s @@ -62,6 +62,7 @@ custom.config: GroupedGemm: False SupportUserArgs: True MatrixInstruction: [16, 16, 16, 1, 1, 4, 14, 4, 1] + WavefrontSize: 64 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s index 63bbea1b4c..c4d3be96af 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x224x64_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s @@ -62,6 +62,7 @@ custom.config: GroupedGemm: False SupportUserArgs: True MatrixInstruction: [16, 16, 16, 1, 1, 4, 14, 4, 1] + WavefrontSize: 64 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.s index c3f9640e04..50d91dc776 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.s @@ -1,47177 +1,47197 @@ - -/******************************************/ -/* Begin Kernel */ -/******************************************/ -.amdgcn_target "amdgcn-amd-amdhsa--gfx942" -.text -.protected Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA -.globl Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA -.p2align 8 -.type Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA,@function -.section .rodata,#alloc -.p2align 6 -.amdhsa_kernel Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA - .amdhsa_user_sgpr_kernarg_segment_ptr 1 - .amdhsa_accum_offset 256 // accvgpr offset - .amdhsa_next_free_vgpr 512 // vgprs - .amdhsa_next_free_sgpr 84 // sgprs - .amdhsa_group_segment_fixed_size 40960 // lds bytes - .amdhsa_private_segment_fixed_size 0 - .amdhsa_system_sgpr_workgroup_id_x 1 - .amdhsa_system_sgpr_workgroup_id_y 1 - .amdhsa_system_sgpr_workgroup_id_z 1 - .amdhsa_system_vgpr_workitem_id 0 - .amdhsa_float_denorm_mode_32 3 - .amdhsa_float_denorm_mode_16_64 3 - .amdhsa_user_sgpr_count 13 - .amdhsa_user_sgpr_kernarg_preload_length 11 - .amdhsa_user_sgpr_kernarg_preload_offset 0 -.end_amdhsa_kernel -.text -/* Num VGPR =256 */ -/* Num AccVGPR=256 */ -/* Num SGPR =84 */ - -/******************************************/ -/* Optimizations and Config: */ -/******************************************/ -/* ThreadTile= 16 x 16 */ -/* SubGroup= 16 x 16 */ -/* VectorWidthA=4 */ -/* VectorWidthB=1 */ -/* GlobalReadVectorWidthA=8, GlobalReadVectorWidthB=8 */ -/* DirectToLdsA=False */ -/* DirectToLdsB=False */ -/* UseSgprForGRO=1 */ -.amdgpu_metadata ---- -custom.config: - ProblemType: - OperationType: GEMM - DataType: b - DestDataType: b - ComputeDataType: s - HighPrecisionAccumulate: True - TransposeA: True - TransposeB: False - UseBias: 1 - Activation: True - UseScaleAlphaVec: 1 - UseBeta: True - Batched: True - GroupedGemm: False - SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4, 16, 4, 1] - 1LDSBuffer: 1 - ScheduleIterAlg: 3 - DepthU: 64 - GlobalReadVectorWidthA: 8 - GlobalReadVectorWidthB: 8 - AssertFree0ElementMultiple: 1 - AssertFree1ElementMultiple: 1 - AssertSummationElementMultiple: 1 - NoReject: True - InternalSupportParams: - KernArgsVersion: 0 - SupportUserGSU: True - SupportCustomWGM: True - SupportCustomStaggerU: True - UseUniversalArgs: True -amdhsa.version: - - 1 - - 1 -amdhsa.kernels: - - .name: Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA - .symbol: 'Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.kd' - .language: OpenCL C - .language_version: - - 2 - - 0 - .args: - - .name: SizesFree0 - .size: 4 - .offset: 0 - .value_kind: by_value - .value_type: u32 - - .name: SizesFree1 - .size: 4 - .offset: 4 - .value_kind: by_value - .value_type: u32 - - .name: SizesFree2 - .size: 4 - .offset: 8 - .value_kind: by_value - .value_type: u32 - - .name: SizesSum0 - .size: 4 - .offset: 12 - .value_kind: by_value - .value_type: u32 - - .name: Gemm info - .size: 4 - .offset: 16 - .value_kind: by_value - .value_type: u32 - - .name: kernel info - .size: 4 - .offset: 20 - .value_kind: by_value - .value_type: u32 - - .name: D - .size: 8 - .offset: 24 - .value_kind: global_buffer - .value_type: bf16 - .address_space: generic - - .name: C - .size: 8 - .offset: 32 - .value_kind: global_buffer - .value_type: bf16 - .address_space: generic - - .name: A - .size: 8 - .offset: 40 - .value_kind: global_buffer - .value_type: bf16 - .address_space: generic - - .name: B - .size: 8 - .offset: 48 - .value_kind: global_buffer - .value_type: bf16 - .address_space: generic - - .name: strideD0 - .size: 4 - .offset: 56 - .value_kind: by_value - .value_type: u32 - - .name: strideD1 - .size: 4 - .offset: 60 - .value_kind: by_value - .value_type: u32 - - .name: strideC0 - .size: 4 - .offset: 64 - .value_kind: by_value - .value_type: u32 - - .name: strideC1 - .size: 4 - .offset: 68 - .value_kind: by_value - .value_type: u32 - - .name: strideA0 - .size: 4 - .offset: 72 - .value_kind: by_value - .value_type: u32 - - .name: strideA1 - .size: 4 - .offset: 76 - .value_kind: by_value - .value_type: u32 - - .name: strideB0 - .size: 4 - .offset: 80 - .value_kind: by_value - .value_type: u32 - - .name: strideB1 - .size: 4 - .offset: 84 - .value_kind: by_value - .value_type: u32 - - .name: alpha - .size: 4 - .offset: 88 - .value_kind: by_value - .value_type: f32 - - .name: beta - .size: 4 - .offset: 92 - .value_kind: by_value - .value_type: f32 - - .name: AddressScaleAlphaVec - .size: 8 - .offset: 96 - .value_kind: global_buffer - .value_type: f32 - .address_space: generic - - .name: bias - .size: 8 - .offset: 104 - .value_kind: global_buffer - .value_type: void - .address_space: generic - - .name: biasType - .size: 4 - .offset: 112 - .value_kind: by_value - .value_type: u32 - - .name: StrideBias - .size: 4 - .offset: 116 - .value_kind: by_value - .value_type: u32 - - .name: activationAlpha - .size: 4 - .offset: 120 - .value_kind: by_value - .value_type: f32 - - .name: activationBeta - .size: 4 - .offset: 124 - .value_kind: by_value - .value_type: f32 - - .name: activationType - .size: 4 - .offset: 128 - .value_kind: by_value - .value_type: u32 - .group_segment_fixed_size: 40960 - .kernarg_segment_align: 8 - .kernarg_segment_size: 136 - .max_flat_workgroup_size: 256 - .private_segment_fixed_size: 0 - .sgpr_count: 84 - .sgpr_spill_count: 0 - .vgpr_count: 256 - .vgpr_spill_count: 0 - .wavefront_size: 64 -... -.end_amdgpu_metadata -Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA: -label_ASM_Start: /// Main body of the asm kernel - -/* Magic div and mod functions */ -.macro V_MAGIC_DIV dstIdx:req dividend:req magicNumber:req magicShift:req magicA:req - v_mul_hi_u32 v[\dstIdx+1] \dividend \magicNumber - v_mul_lo_u32 v[\dstIdx+0] \dividend \magicA - v_add_u32 v[\dstIdx+0] v[\dstIdx+0] v[\dstIdx+1] - v_lshrrev_b32 v[\dstIdx+0] \magicShift v[\dstIdx+0] -.endm - -/******************************************/ -/* VGPR Assignments */ -/******************************************/ -/* ValuC range: [0-0), serializedStore enabled */ -.set vgprValuC, 0 -/* ValuA/B Xn=PLR buffer idx, In=InnerUnroll idx */ -.set vgprValuA_X0_I0_0, 0 -.set vgprValuA_X2_I0_0, 16 -.set vgprValuB_X0_I0, 32 -.set vgprValuB_X2_I0, 96 -.set vgprLocalWriteAddrA, 160 -.set vgprLocalWriteAddrB, 161 -.set vgprGlobalReadOffsetA, 162 -.set vgprGlobalReadOffsetB, 163 -.set vgprG2LB, 164 -.set vgprValuA_X0_I0_1, 196 -.set vgprValuA_X2_I0_1, 212 -.set vgprLocalReadAddrA, 228 -.set vgprLocalReadAddrB, 229 -.set vgprSerial, 230 - -/******************************************/ -/* SGPR Assignments */ -/******************************************/ -.set sgprKernArgAddress, 0 -.set sgprWorkGroup0, 2 -.set sgprWorkGroup1, 3 -.set sgprWorkGroup2, 4 -.set sgprArgType, 5 -.set sgprGSUSumIdx, 6 -.set sgprGSULog2BpeC, 8 -.set sgprGSULog2BpeD, 9 -.set sgprStaggerU, 10 -.set sgprWGM, 11 -.set sgprLoopCounterL, 12 -.set sgprOrigLoopCounter, 13 -.set sgprSrdD, 16 -.set sgprSrdC, 20 -.set sgprNumWorkGroups0, 14 -.set sgprNumWorkGroups1, 15 -.set sgprSizesFree, 24 -.set sgprSizesSum, 27 -.set sgprAddressD, 28 -.set sgprAddressC, 30 -.set sgprAddressA, 32 -.set sgprAddressB, 34 -.set sgprStridesD, 36 -.set sgprStridesC, 38 -.set sgprStridesA, 40 -.set sgprStridesB, 42 -.set sgprAlpha, 44 -.set sgprBeta, 45 -.set sgprGSU, 46 - -/* Size Assignments */ -.set sgprSizeI, sgprSizesFree+0 -.set sgprSizeJ, sgprSizesFree+1 -.set sgprSizeK, sgprSizesFree+2 -.set sgprSizeL, sgprSizesSum+0 - -/* Stride Assignments */ -.set constStrideD0I, 1 -.set sgprStrideD1J, sgprStridesD+0 -.set sgprStrideDK, sgprStridesD+1 -.set constStrideC0I, 1 -.set sgprStrideC1J, sgprStridesC+0 -.set sgprStrideCK, sgprStridesC+1 -.set constStrideAL, 1 -.set sgprStrideA0I, sgprStridesA+0 -.set sgprStrideAK, sgprStridesA+1 -.set constStrideBL, 1 -.set sgprStrideB1J, sgprStridesB+0 -.set sgprStrideBK, sgprStridesB+1 - -.set MT0, 256 -.set MT1, 256 -.set DepthU, 64 -.set BpeA, 2 -.set BpeALog2, 1 -.set BpeB, 2 -.set BpeBLog2, 1 -.set BpeAGR, 2 -.set BpeAGRLog2, 1 -.set BpeBGR, 2 -.set BpeBGRLog2, 1 -/* Number of elements to shift-left SRD */ -.set SrdShiftLeftA, 8 -.set SrdShiftLeftB, 8 -/* 2GB limit - set offsets to -1 to exceed this and clamp */ -.set BufferLimit, 0xffffffff -.set BufferOOB, 0x80000000 - -/******************************************/ -/* Bits 127:96 of SRD. */ -/* hex: 0x00020000 */ -/* dst_sel_x (3b): 0 */ -/* dst_sel_y (3b): 0 */ -/* dst_sel_z (3b): 0 */ -/* dst_sel_w (3b): 0 */ -/* num_format (3b): 0 */ -/* data_format (4b): 4 */ -/* user_vm_enable (1b): 0 */ -/* user_vm_mode (1b): 0 */ -/* index_stride (2b): 0 */ -/* add_tid_enable (1b): 0 */ -/* _unusedA (3b): 0 */ -/* nv (1b): 0 */ -/* _unusedB (2b): 0 */ -/* type (2b): 0 */ -/******************************************/ -.set Srd127_96, 0x00020000 - -/* Global Offset A */ -.macro GLOBAL_OFFSET_A vgprAddr:req vgprTmp:req - v_and_b32 v[\vgprTmp+0], 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) - v_and_b32 v[\vgprAddr+0], 15, v[\vgprTmp+0] // 1. M offset: mIdx = wtid % MI_M(16) - v_mul_lo_u32 v[\vgprAddr+0], s[sgprStrideA0I], v[\vgprAddr+0] // 1. M offset: mOffset = mIdx * mStride(k) - v_lshlrev_b32 v[\vgprAddr+0], 0x2, v[\vgprAddr+0] // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) - v_and_b32 v[\vgprTmp+0], 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) - v_lshrrev_b32 v[\vgprTmp+0], 4, v[\vgprTmp+0] // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) - v_lshlrev_b32 v[\vgprTmp+0], 0x3, v[\vgprTmp+0] // 5. K offset: lrKOffset = kIdx * mStride(8) - v_add_u32 v[\vgprAddr+0], v[\vgprTmp+0], v[\vgprAddr+0] // 6. offset in wave: lrOffset = bnOffset + lrKOffset - v_lshrrev_b32 v[\vgprTmp+0], 6, v[vgprSerial] // 7. wave offset in M dimen: wtid = tid / dividedForWaveId(64) - v_and_b32 v[\vgprTmp+0], 3, v[\vgprTmp+0] // 7. wave offset in M dimen: wtid0 = wtid % num1DWaves(4) - v_mul_lo_u32 v[\vgprTmp+0], s[sgprStrideA0I], v[\vgprTmp+0] // 7. wave offset in M dimen: wOffset = wtid0 * s[sgprStrideA0I](8192) - v_lshlrev_b32 v[\vgprTmp+0], 0x6, v[\vgprTmp+0] // 7. wave offset in M dimen: wOffset = wOffset * 16 * vw(4) - v_add_u32 v[\vgprAddr+0], v[\vgprTmp+0], v[\vgprAddr+0] // 7. final local read offset: flrOffset = lrOffset + WOffset - v_add_u32 v[\vgprAddr+0] 0x8 v[\vgprAddr+0] // add prepad for pointer shift - v_lshlrev_b32 v[\vgprAddr+0] 0x1 v[\vgprAddr+0] // offset *= bytes/element -.endm - -/* Global Offset B */ -.macro GLOBAL_OFFSET_B vgprAddr:req vgprOffsetL:req vgprOffset1J:req vgprTmp:req - v_mul_lo_u32 v[\vgprTmp+0] s[sgprStrideB1J] v[\vgprOffset1J] // mul d1 lower - v_add_co_u32 v[\vgprAddr+0] vcc v[\vgprOffsetL] v[\vgprTmp+0] // accumulate K lower - v_add_u32 v[\vgprAddr+0] 0x8 v[\vgprAddr+0] // add prepad for pointer shift - v_lshlrev_b32 v[\vgprAddr+0] 0x1 v[\vgprAddr+0] // offset *= bytes/element -.endm - -/* Dynamic Scalar Divide: vQuotient=vDividend/vDivisor; vRemainder=vDividend%vDivisor; */ -.macro DYNAMIC_VECTOR_DIVIDE vQuotient vRemainder vDividend vDivisor vTmp0 vTmp1 sTmp - v_cvt_f32_u32 v[\vQuotient] v[\vDivisor] - v_rcp_f32 v[\vQuotient] v[\vQuotient] - v_mul_f32 v[\vQuotient] 0x4f800000 v[\vQuotient] - v_cvt_u32_f32 v[\vQuotient] v[\vQuotient] - v_mul_lo_u32 v[\vRemainder] v[\vDivisor] v[\vQuotient] - v_mul_hi_u32 v[\vTmp0] v[\vDivisor] v[\vQuotient] - v_sub_co_u32 v[\vTmp1] vcc 0x0 v[\vRemainder] - v_cmp_ne_i32 s[\sTmp:\sTmp+1] 0x0 v[\vTmp0] - v_cndmask_b32 v[\vRemainder] v[\vTmp1] v[\vRemainder] s[\sTmp:\sTmp+1] - v_mul_hi_u32 v[\vRemainder] v[\vRemainder] v[\vQuotient] - v_sub_co_u32 v[\vTmp0] vcc v[\vQuotient] v[\vRemainder] - v_add_co_u32 v[\vQuotient] vcc v[\vQuotient] v[\vRemainder] - v_cndmask_b32 v[\vQuotient] v[\vQuotient] v[\vTmp0] s[\sTmp:\sTmp+1] - v_mul_hi_u32 v[\vQuotient] v[\vQuotient] v[\vDividend] - v_mul_lo_u32 v[\vRemainder] v[\vQuotient] v[\vDivisor] - v_sub_co_u32 v[\vTmp0] vcc v[\vDividend] v[\vRemainder] - v_cmp_ge_u32 s[\sTmp:\sTmp+1] v[\vDividend] v[\vRemainder] - v_add_co_u32 v[\vRemainder] vcc 0x1 v[\vQuotient] - v_add_co_u32 v[\vTmp1] vcc -1 v[\vQuotient] - v_cmp_le_u32 vcc v[\vDivisor] v[\vTmp0] - s_and_b64 vcc s[\sTmp:\sTmp+1] vcc - v_cndmask_b32 v[\vQuotient] v[\vQuotient] v[\vRemainder] vcc - v_cndmask_b32 v[\vQuotient] v[\vTmp1] v[\vQuotient] s[\sTmp:\sTmp+1] - v_cmp_ne_i32 vcc 0x0 v[\vDivisor] - v_cndmask_b32 v[\vQuotient] -1 v[\vQuotient] vcc // final result - v_mul_lo_u32 v[\vRemainder] v[\vQuotient] v[\vDivisor] - v_sub_co_u32 v[\vRemainder] vcc v[\vDividend] v[\vRemainder] // final result -.endm - -/******************************************/ -/* Allocate Resources */ -/******************************************/ - -/* Load num of Gemms */ -s_load_dword s47, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x0 - -/* Load GSU data */ -s_load_dword s[sgprGSU], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x4 -s_waitcnt lgkmcnt(0) -s_lshr_b32 s48, s47, 0x1e // Get arg type -s_and_b32 s47, 0x3fffffff, s47 // Get nums of gemm -s_cmp_eq_u32 s48, 0 // Is kernel args -s_cbranch_scc0 label_HBMArgs -s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], 0x8 // Shift common args -s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 - -/* Load Kernel Args */ -s_load_dwordx16 s[24:39], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x0 -s_load_dwordx4 s[40:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x40 -s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 -s_waitcnt lgkmcnt(0) -s_branch label_LoadArgsEnd -label_HBMArgs: - -/* Load address of kernel arguments */ -s_load_dwordx2 s[sgprKernArgAddress:sgprKernArgAddress+1], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x8 -s_waitcnt lgkmcnt(0) // wait for args to load -label_LoadArgsEnd: -s_branch label_common_kernel_entry - -/* pad 41 snops to satisfy 0x100 code size for Preload Backward Compatibility Prologue */ -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -s_nop 0 -label_Preload_Offset_Start: -s_and_b32 s47, 0x3fffffff, s2 // Get nums of gemm -s_lshr_b32 s48, s2, 0x1e // Get arg type -s_mov_b32 s[sgprGSU], s3 // Preload internal args -s_cmp_eq_u32 s48, 0 // Is kernel args -s_cbranch_scc0 label_Preload_HBMArgs -s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], 0x8 // Shift common args -s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 - -/* Load Kernel Args */ -s_load_dword s33, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x24 -s_load_dwordx2 s[34:35], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x28 -s_load_dwordx8 s[36:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x30 -s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 -s_mov_b32 s24, s4 // move preload data to correct sgpr -s_mov_b32 s25, s5 // move preload data to correct sgpr -s_mov_b32 s26, s6 // move preload data to correct sgpr -s_mov_b32 s27, s7 // move preload data to correct sgpr -s_mov_b32 s28, s8 // move preload data to correct sgpr -s_mov_b32 s29, s9 // move preload data to correct sgpr -s_mov_b32 s30, s10 // move preload data to correct sgpr -s_mov_b32 s31, s11 // move preload data to correct sgpr -s_mov_b32 s32, s12 // move preload data to correct sgpr -s_branch label_Preload_LoadArgsEnd -label_Preload_HBMArgs: -s_mov_b64 s[sgprKernArgAddress:sgprKernArgAddress+1], s[4:5] // Load address of kernel arguments -label_Preload_LoadArgsEnd: -label_common_kernel_entry: /// for both preload/non-preload common code -s_mov_b32 s[sgprWorkGroup0+0], s13 // restore workgroup id -s_mov_b32 s[sgprWorkGroup0+1], s14 // restore workgroup id -s_mov_b32 s[sgprWorkGroup0+2], s15 // restore workgroup id -s_and_b32 s[sgprWGM], s[sgprGSU], 0xff00 // Restore WGM -s_lshr_b32 s[sgprWGM], s[sgprWGM], 0x8 -s_and_b32 s[sgprStaggerU], s[sgprGSU], 0xffff0000 // Restore StaggerU related vars -s_lshr_b32 s[sgprStaggerU], s[sgprStaggerU], 0x10 -s_and_b32 s[sgprGSU], s[sgprGSU], 0xff // Restore GSU -s_mov_b32 s[sgprArgType], s48 -s_mov_b32 m0, 0xa000 // LDS clamp at 40960 bytes -v_mov_b32 v[vgprSerial], v0 // thread serial id -s_cmp_eq_u32 s48, 0 -s_cbranch_scc0 label_MultiGemm -/* init: add vgpr [0...160) to pool */ -/* init: add vgpr [0...0) to pool */ -/* init: add agpr [0...256) to pool */ - -/******************************************/ -/* Local Read Addresses */ -/******************************************/ - -/* local read addresses: tile assignments a/b */ -/* lr0I */ -v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v0, 15, v1 // 1. N offset: nIdx = wtid % MI_N(16) -v_lshlrev_b32 v0, 0x6, v0 // 1. N offset: nOffset = nIdx * nStride(64) -/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ -v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) -v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v1, 0x3, v1 // 5. K offset: lrKOffset = kIdx * mStride(8) -v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset -v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in N dimen: wtid = tid / dividedForWaveId(64) -v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) -v_lshlrev_b32 v1, 0xc, v1 // 7. wave offset in M dimen: wOffset = wtid0 * W0Stride(4096) -v_add_u32 v0, v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset -/* lr1J */ -v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) -v_lshlrev_b32 v1, 0x6, v1 // 1. N offset: nOffset = nIdx * nStride(64) -/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ - // 4. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) -v_and_b32 v2, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v2, 4, v2 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v2, 0x3, v2 // 5. K offset: lrKOffset = kIdx * mStride(8) -v_add_u32 v1, v2, v1 // 6. offset in wave: lrOffset = bnOffset + lrKOffset - -/* local read addresses: final offsets a */ -v_lshrrev_b32 v2, 6, v[vgprSerial] // v2 = v[vgprSerial] / 64 -v_lshrrev_b32 v2, 2, v2 // LSU offset: Get LSU wave_id -s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True -v_mul_lo_u32 v2, s49, v2 // LSU offset: lsuoffset = wave_id*lsuStride*(MT0+PAD) -v_add_lshl_u32 v[vgprLocalReadAddrA], v2, v0, 0x1 // Final Offset: offset = (lro0+lsuoffset)*bpeDS -v_lshrrev_b32 v3, 9, v[vgprLocalReadAddrA] // Final Offset: padding 32 per block 512 -v_lshlrev_b32 v3, 0x5, v3 // Final Offset: padding 32 per block 512 -v_add_u32 v[vgprLocalReadAddrA], v3, v[vgprLocalReadAddrA] // Final Offset: add padding 32 per block 512 - -/* local read addresses: final offsets b */ -v_lshrrev_b32 v0, 6, v[vgprSerial] // v0 = v[vgprSerial] / 64 -v_lshrrev_b32 v0, 2, v0 // LSU offset: Get LSU wave_id -s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True -v_mul_lo_u32 v0, s49, v0 // LSU offset: lsuoffset = wave_id*lsuStride*(MT1+PAD) -v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v1, 0x1 // Final Offset: offset = (lro1+lsuoffset)*bpeDS -v_lshrrev_b32 v2, 7, v[vgprLocalReadAddrB] // Final Offset: padding 32 per block 128 -v_lshlrev_b32 v2, 0x5, v2 // Final Offset: padding 32 per block 128 -v_add_u32 v[vgprLocalReadAddrB], v2, v[vgprLocalReadAddrB] // Final Offset: add padding 32 per block 128 - -/* local read addresses: declare addresses a */ -/* N/A */ - -/* local read addresses: declare addresses b */ - -/******************************************/ -/* Local Write Addresses */ -/******************************************/ -/* LVCA = 8 */ -/* v1 = A-unroll = serial%LVCA */ -v_lshrrev_b32 v0, 3, v[vgprSerial] // v0 = v[vgprSerial] / 8 -v_and_b32 v1, 7, v[vgprSerial] // v1 = v[vgprSerial] % 8 -/* unroll *= glvw */ -v_lshlrev_b32 v1, 0x3, v1 // v1 = v1 * 8 -v_mov_b32 v4, v1 // copy for GlobalSplitU -/* LVCB = 8 */ -/* v3 = B-unroll = serial%LVCB */ -v_lshrrev_b32 v2, 3, v[vgprSerial] // v2 = v[vgprSerial] / 8 -v_and_b32 v3, 7, v[vgprSerial] // v3 = v[vgprSerial] % 8 -/* unroll *= glvw */ -v_lshlrev_b32 v3, 0x3, v3 // v3 = v3 * 8 -v_mov_b32 v5, v3 // copy for GlobalSplitU -/* lwaUnrollAssignmentA = v4 */ -/* lwaUnrollAssignmentB = v5 */ - -/* local write addresses: first offset a */ -v_mul_u32_u24 v[vgprLocalWriteAddrA], 0x40, v0 // lwAL**(DepthU_Compute + PAD) -v_add_lshl_u32 v[vgprLocalWriteAddrA], v4, v[vgprLocalWriteAddrA], 0x1 // lwFOA = (lwAA + lwAL*(DepthU+PAD))*bpeDS -v_lshrrev_b32 v6, 9, v[vgprLocalWriteAddrA] // padding 32 per block 512 -v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 512 -v_add_u32 v[vgprLocalWriteAddrA], v6, v[vgprLocalWriteAddrA] // add padding 32 per block 512 - -/* local write addresses: first offset b */ -v_mul_u32_u24 v[vgprLocalWriteAddrB], 0x40, v2 // lwBL**(DepthU_Compute + PAD) -v_add_lshl_u32 v[vgprLocalWriteAddrB], v5, v[vgprLocalWriteAddrB], 0x1 // lwFOB = (lwBB + lwBL*(DepthU+PAD))*bpeDS -v_lshrrev_b32 v6, 7, v[vgprLocalWriteAddrB] // padding 32 per block 128 -v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 128 -v_add_u32 v[vgprLocalWriteAddrB], v6, v[vgprLocalWriteAddrB] // add padding 32 per block 128 -v_mov_b32 v8, MT0 // set MT0 into sgpr -v_mov_b32 v7, s[sgprSizesFree+0] // set Free0 size -v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) -v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) -v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) -v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) -v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) -v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) -v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) -v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) -v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil -v_mov_b32 v8, MT1 // set MT1 into sgpr -v_mov_b32 v7, s[sgprSizesFree+1] // set Free1 size -v_readfirstlane_b32 s[sgprNumWorkGroups0], v6 // set back to numWorkGroup0 -v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) -v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) -v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) -v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) -v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) -v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) -v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) -v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) -v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil -s_nop 0 // 1 wait states -v_readfirstlane_b32 s[sgprNumWorkGroups1], v6 // set back to numWorkGroup1 -s_waitcnt lgkmcnt(0) // wait for 44/0 bytes of kern args -s_branch label_MultiGemmEnd -label_MultiGemm: - -/* Check if custom structure pointer is null */ -s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? -s_cbranch_scc1 label_IsExternalValid // branch if ArgType == 2 -s_mov_b32 s15, 124 -s_mul_i32 s54, s47, 4 -s_mov_b64 s[48:49], s[sgprKernArgAddress:sgprKernArgAddress+1] -s_branch label_IsExternalValidEnd -label_IsExternalValid: -s_mov_b32 s15, 196 -s_mov_b32 s54, 0x0 -s_mov_b64 s[48:49], s[sgprKernArgAddress:sgprKernArgAddress+1] -label_IsExternalValidEnd: - -/* Grouped Gemm:: prefetch 1 arg load */ -s_mov_b32 s14, 1 -s_mov_b32 s55, 0 -s_load_dwordx4 s[24:27], s[48:49], s54 -s_cmpk_eq_u32 s47, 1 // if gemm_count is 1? -s_cbranch_scc1 label_wgTable_noLoadLoop - -/* Grouped Gemm:: accumulate numTiles for each gemm */ -/* Grouped Gemm:: loop start */ -label_Loop_GemmCount: -s_waitcnt lgkmcnt(0) -s_lshr_b32 s52, s24, 8 // s52 = s24 / 256 -s_and_b32 s50, 255, s24 // s50 = s24 % 256 -s_addc_u32 s52, s52, 0x0 -s_lshr_b32 s53, s25, 8 // s53 = s25 / 256 -s_and_b32 s50, 255, s25 // s50 = s25 % 256 -s_addc_u32 s53, s53, 0x0 -s_mul_i32 s52, s52, s53 -s_mul_i32 s52, s52, s26 -s_mul_i32 s52, s52, s[sgprGSU] -s_add_u32 s55, s55, s52 -s_cmp_lt_u32 s[sgprWorkGroup0], s55 -s_cbranch_scc1 label_FOUND -s_add_u32 s54, s54, s15 -s_load_dwordx4 s[24:27], s[48:49], s54 -s_add_u32 s14, s14, 1 -s_cmp_lt_u32 s14, s47 -s_cbranch_scc1 label_Loop_GemmCount - -/* Grouped Gemm:: noLoadLoop */ -label_wgTable_noLoadLoop: -s_waitcnt lgkmcnt(0) -s_lshr_b32 s52, s24, 8 // s52 = s24 / 256 -s_and_b32 s50, 255, s24 // s50 = s24 % 256 -s_addc_u32 s52, s52, 0x0 -s_lshr_b32 s53, s25, 8 // s53 = s25 / 256 -s_and_b32 s50, 255, s25 // s50 = s25 % 256 -s_addc_u32 s53, s53, 0x0 -s_mul_i32 s52, s52, s53 -s_mul_i32 s52, s52, s26 -s_mul_i32 s52, s52, s[sgprGSU] -s_add_u32 s55, s55, s52 - -/* Grouped Gemm:: gemmIndex found */ -label_FOUND: -s_sub_u32 s49, s14, 1 -s_sub_u32 s48, s55, s52 -s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s48 -/* Check if custom structure pointer is null */ -s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? -s_cbranch_scc1 label_LoadExternalStruct // branch if ArgType == 2 - -/* Grouped Gemm: offset argument address to gemm */ -/* Grouped Gemm: offset address from wg_table_start to args_start */ -s_lshl2_add_u32 s[sgprKernArgAddress], s47, s[sgprKernArgAddress] -s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 -/* Grouped Gemm: offset address from args_start to gemm_start */ -s_mul_i32 s49, s49, 124 -s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], s49 -s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 - -/* Load Kernel Args */ -s_load_dwordx16 s[28:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x10 -s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 -s_branch label_LoadExternalStructEnd -label_LoadExternalStruct: -/* Grouped Gemm: offset address from args_start to gemm_start */ -s_mul_i32 s49, s49, 196 -s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], s49 -s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 -s_load_dwordx16 s[28:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x10 -s_load_dword s44, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 -// Read Beta -s_load_dword s45, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x60 -label_LoadExternalStructEnd: -/* init: add vgpr [0...160) to pool */ -/* init: add vgpr [0...0) to pool */ -/* init: add agpr [0...256) to pool */ - -/******************************************/ -/* Local Read Addresses */ -/******************************************/ - -/* local read addresses: tile assignments a/b */ -/* lr0I */ -v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v0, 15, v1 // 1. N offset: nIdx = wtid % MI_N(16) -v_lshlrev_b32 v0, 0x6, v0 // 1. N offset: nOffset = nIdx * nStride(64) -/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ -v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) -v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v1, 0x3, v1 // 5. K offset: lrKOffset = kIdx * mStride(8) -v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset -v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in N dimen: wtid = tid / dividedForWaveId(64) -v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) -v_lshlrev_b32 v1, 0xc, v1 // 7. wave offset in M dimen: wOffset = wtid0 * W0Stride(4096) -v_add_u32 v0, v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset -/* lr1J */ -v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) -v_lshlrev_b32 v1, 0x6, v1 // 1. N offset: nOffset = nIdx * nStride(64) -/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ - // 4. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) -v_and_b32 v2, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v2, 4, v2 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v2, 0x3, v2 // 5. K offset: lrKOffset = kIdx * mStride(8) -v_add_u32 v1, v2, v1 // 6. offset in wave: lrOffset = bnOffset + lrKOffset - -/* local read addresses: final offsets a */ -v_lshrrev_b32 v2, 6, v[vgprSerial] // v2 = v[vgprSerial] / 64 -v_lshrrev_b32 v2, 2, v2 // LSU offset: Get LSU wave_id -s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True -v_mul_lo_u32 v2, s49, v2 // LSU offset: lsuoffset = wave_id*lsuStride*(MT0+PAD) -v_add_lshl_u32 v[vgprLocalReadAddrA], v2, v0, 0x1 // Final Offset: offset = (lro0+lsuoffset)*bpeDS -v_lshrrev_b32 v3, 9, v[vgprLocalReadAddrA] // Final Offset: padding 32 per block 512 -v_lshlrev_b32 v3, 0x5, v3 // Final Offset: padding 32 per block 512 -v_add_u32 v[vgprLocalReadAddrA], v3, v[vgprLocalReadAddrA] // Final Offset: add padding 32 per block 512 - -/* local read addresses: final offsets b */ -v_lshrrev_b32 v0, 6, v[vgprSerial] // v0 = v[vgprSerial] / 64 -v_lshrrev_b32 v0, 2, v0 // LSU offset: Get LSU wave_id -s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True -v_mul_lo_u32 v0, s49, v0 // LSU offset: lsuoffset = wave_id*lsuStride*(MT1+PAD) -v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v1, 0x1 // Final Offset: offset = (lro1+lsuoffset)*bpeDS -v_lshrrev_b32 v2, 7, v[vgprLocalReadAddrB] // Final Offset: padding 32 per block 128 -v_lshlrev_b32 v2, 0x5, v2 // Final Offset: padding 32 per block 128 -v_add_u32 v[vgprLocalReadAddrB], v2, v[vgprLocalReadAddrB] // Final Offset: add padding 32 per block 128 - -/* local read addresses: declare addresses a */ -/* N/A */ - -/* local read addresses: declare addresses b */ - -/******************************************/ -/* Local Write Addresses */ -/******************************************/ -/* LVCA = 8 */ -/* v1 = A-unroll = serial%LVCA */ -v_lshrrev_b32 v0, 3, v[vgprSerial] // v0 = v[vgprSerial] / 8 -v_and_b32 v1, 7, v[vgprSerial] // v1 = v[vgprSerial] % 8 -/* unroll *= glvw */ -v_lshlrev_b32 v1, 0x3, v1 // v1 = v1 * 8 -v_mov_b32 v4, v1 // copy for GlobalSplitU -/* LVCB = 8 */ -/* v3 = B-unroll = serial%LVCB */ -v_lshrrev_b32 v2, 3, v[vgprSerial] // v2 = v[vgprSerial] / 8 -v_and_b32 v3, 7, v[vgprSerial] // v3 = v[vgprSerial] % 8 -/* unroll *= glvw */ -v_lshlrev_b32 v3, 0x3, v3 // v3 = v3 * 8 -v_mov_b32 v5, v3 // copy for GlobalSplitU -/* lwaUnrollAssignmentA = v4 */ -/* lwaUnrollAssignmentB = v5 */ - -/* local write addresses: first offset a */ -v_mul_u32_u24 v[vgprLocalWriteAddrA], 0x40, v0 // lwAL**(DepthU_Compute + PAD) -v_add_lshl_u32 v[vgprLocalWriteAddrA], v4, v[vgprLocalWriteAddrA], 0x1 // lwFOA = (lwAA + lwAL*(DepthU+PAD))*bpeDS -v_lshrrev_b32 v6, 9, v[vgprLocalWriteAddrA] // padding 32 per block 512 -v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 512 -v_add_u32 v[vgprLocalWriteAddrA], v6, v[vgprLocalWriteAddrA] // add padding 32 per block 512 - -/* local write addresses: first offset b */ -v_mul_u32_u24 v[vgprLocalWriteAddrB], 0x40, v2 // lwBL**(DepthU_Compute + PAD) -v_add_lshl_u32 v[vgprLocalWriteAddrB], v5, v[vgprLocalWriteAddrB], 0x1 // lwFOB = (lwBB + lwBL*(DepthU+PAD))*bpeDS -v_lshrrev_b32 v6, 7, v[vgprLocalWriteAddrB] // padding 32 per block 128 -v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 128 -v_add_u32 v[vgprLocalWriteAddrB], v6, v[vgprLocalWriteAddrB] // add padding 32 per block 128 -v_mov_b32 v8, MT0 // set MT0 into sgpr -v_mov_b32 v7, s[sgprSizesFree+0] // set Free0 size -v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) -v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) -v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) -v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) -v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) -v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) -v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) -v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) -v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil -v_mov_b32 v8, MT1 // set MT1 into sgpr -v_mov_b32 v7, s[sgprSizesFree+1] // set Free1 size -v_readfirstlane_b32 s[sgprNumWorkGroups0], v6 // set back to numWorkGroup0 -v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) -v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) -v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) -v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) -v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) -v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) -v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) -v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) -v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil -s_nop 0 // 1 wait states -v_readfirstlane_b32 s[sgprNumWorkGroups1], v6 // set back to numWorkGroup1 -s_waitcnt lgkmcnt(0) // wait for 44/0 bytes of kern args - -/* Early stop if N(SizeFreeJ) == 0 */ -s_cmp_eq_u32 s[sgprSizeJ], 0x0 -s_cbranch_scc0 label_NoEarlyStop_N0 -label_EarlyStop_if_N_is_0: -s_endpgm -label_NoEarlyStop_N0: - -/* Grouped Gemm: remap wg from 1D(idxWG012) to 3D(wg2,wg1,wg0) */ -/* wg2 = idxWG012 * smallMagicNumber(1/(numWG0*numWG1)) */ -s_mul_i32 s48, s[sgprNumWorkGroups0], s[sgprNumWorkGroups1] -s_mul_i32 s48, s48, s[sgprGSU] -v_cvt_f32_u32 v6, s48 // s48 = s[sgprWorkGroup0] / s48 -v_rcp_iflag_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s48 -v_cvt_f32_u32 v7, s[sgprWorkGroup0] // s48 = s[sgprWorkGroup0] / s48 -v_mul_f32 v6, v6, v7 // s48 = s[sgprWorkGroup0] / s48 -v_cvt_u32_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s48 -v_mul_u32_u24 v7, v6, s48 // s48 = s[sgprWorkGroup0] / s48 -v_sub_u32 v7, s[sgprWorkGroup0], v7 // s48 = s[sgprWorkGroup0] / s48 -v_cmpx_eq_u32 exec, v7, s48 // s48 = s[sgprWorkGroup0] / s48 -v_add_u32 v6, 1, v6 // s48 = s[sgprWorkGroup0] / s48 -s_mov_b64 exec, -1 // s48 = s[sgprWorkGroup0] / s48 -v_readfirstlane_b32 s48, v6 -s_mov_b32 s[sgprWorkGroup2], s48 -/* idxWG01 = idxWG012 - wg2 * numWG0 * numWG1 */ -s_mul_i32 s48, s[sgprNumWorkGroups1], s[sgprNumWorkGroups0] -s_mul_i32 s48, s48, s[sgprWorkGroup2] -s_mul_i32 s48, s48, s[sgprGSU] -s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s48 -/* wg1 = idxWG01 * smallMagicNumber(1/numWG0) */ -v_cvt_f32_u32 v6, s[sgprNumWorkGroups0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_rcp_iflag_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_cvt_f32_u32 v7, s[sgprWorkGroup0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_mul_f32 v6, v6, v7 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_cvt_u32_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_mul_u32_u24 v7, v6, s[sgprNumWorkGroups0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_sub_u32 v7, s[sgprWorkGroup0], v7 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_cmpx_eq_u32 exec, v7, s[sgprNumWorkGroups0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_add_u32 v6, 1, v6 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -s_mov_b64 exec, -1 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] -v_readfirstlane_b32 s48, v6 -s_mov_b32 s[sgprWorkGroup1], s48 -/* wg0 = idxWG01 - wg1 * numWG0 */ -s_mul_i32 s48, s[sgprWorkGroup1], s[sgprNumWorkGroups0] -s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s48 - -/* Early stop if wg exceed */ -s_cmp_ge_u32 s[sgprWorkGroup2], s[sgprSizesFree+2] -s_cbranch_scc0 label_NoEarlyStop_wgExceed -label_EarlyStop_if_wg_exceed: -s_endpgm -label_NoEarlyStop_wgExceed: - -label_MultiGemmEnd: -.set sgprSrdA, 48 -.set sgprSrdB, 52 -.set sgprShadowLimitA, 56 -.set sgprShadowLimitB, 58 -.set sgprStaggerUIter, 47 -.set sgprWrapUA, 60 -.set sgprWrapUB, 62 -.set sgprGlobalReadIncsA, 64 -.set sgprGlobalReadIncsB, 65 -.set sgprScalarGlobalReadOffsetA, 66 -.set sgprScalarGlobalReadOffsetB, 73 -s_sub_u32 s[sgprAddressA+0], s[sgprAddressA+0], 16 // pre-pad to make room for possible pointer shift -s_subb_u32 s[sgprAddressA+1], s[sgprAddressA+1], 0 // pre-pad to make room for possible pointer shift -s_sub_u32 s[sgprAddressB+0], s[sgprAddressB+0], 16 // pre-pad to make room for possible pointer shift -s_subb_u32 s[sgprAddressB+1], s[sgprAddressB+1], 0 // pre-pad to make room for possible pointer shift - -/* Short circuit condition if Alpha == 0, then sumDims=0 */ -v_cmp_eq_f32 vcc, s[sgprAlpha], 0.0 // s[Alpha] == 0.0f ? -s_cbranch_vccz label_AlphaNonZero // branch if s[Alpha] != 0 -s_mov_b32 s[sgprSizesSum+0], 0x0 // Set summation dim=0 if Alpha == 0 -label_AlphaNonZero: - -/******************************************/ -/* Begin setupNewTile */ -/******************************************/ - -/* global read addresses: work-group */ -/* graWorkGroup mapping */ -s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? -s_cbranch_scc1 label_GSU // branch if GSU == 1 -// GSU-not-WGMapRR :nwg1 = (size1J + MT1J - 1) / MT1J; -v_cvt_f32_u32 v6, s[sgprGSU] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_rcp_iflag_f32 v6, v6 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_cvt_f32_u32 v7, s[sgprWorkGroup1] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_mul_f32 v6, v6, v7 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_cvt_u32_f32 v6, v6 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_mul_u32_u24 v7, v6, s[sgprGSU] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_sub_u32 v7, s[sgprWorkGroup1], v7 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_cmpx_eq_u32 exec, v7, s[sgprGSU] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_add_u32 v6, 1, v6 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_mov_b32 v7, 0 // s[sgprGSUSumIdx] = s[sgprWorkGroup1] % s[sgprGSU] -s_mov_b64 exec, -1 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] -v_readfirstlane_b32 s[sgprWorkGroup1], v6 -v_readfirstlane_b32 s[sgprGSUSumIdx], v7 -s_mov_b32 s[sgprGSULog2BpeC], 1 -s_mov_b32 s[sgprGSULog2BpeD], 2 -s_branch label_GSU_End -label_GSU: -s_mov_b64 s[sgprGSUSumIdx:sgprGSUSumIdx+1], 0 // Set GSUSumIdx to 0 -s_mov_b32 s[sgprGSULog2BpeC], 1 -s_mov_b32 s[sgprGSULog2BpeD], 1 -label_GSU_End: -s_cmp_le_u32 s[sgprWGM], 1 // WGM <= 1 ? -s_cbranch_scc1 label_WGM // branch if WGM <= 1 -v_cvt_f32_u32 v6, s[sgprWGM] // WGM -v_rcp_iflag_f32 v6, v6 // WGM -v_cvt_f32_u32 v7, s[sgprWorkGroup1] // WGM -v_mul_f32 v6, v6, v7 // WGM -v_cvt_u32_f32 v6, v6 // WGM -v_mul_u32_u24 v7, v6, s[sgprWGM] // WGM -v_sub_u32 v7, s[sgprWorkGroup1], v7 // WGM -v_cmpx_eq_u32 exec, v7, s[sgprWGM] // WGM -v_add_u32 v6, 1, v6 // WGM -s_mov_b64 exec, -1 // WGM -v_readfirstlane_b32 s82, v6 -s_mul_i32 s83, s82, s[sgprWGM] // quotient * non-magic divisor -s_sub_u32 s83, s[sgprWorkGroup1], s83 // WorkGroup1=remainder -s_mul_i32 s83, s83, s[sgprNumWorkGroups0] // (wg1 % WGM)*nwg0 -s_add_u32 s83, s83, s[sgprWorkGroup0] // wgSerial = wg0 + (wg1 % WGM)*nwg0 -v_cvt_f32_u32 v6, s[sgprWGM] // WGM -v_rcp_iflag_f32 v6, v6 // WGM -v_cvt_f32_u32 v7, s[sgprNumWorkGroups1] // WGM -v_mul_f32 v6, v6, v7 // WGM -v_cvt_u32_f32 v6, v6 // WGM -v_mul_u32_u24 v7, v6, s[sgprWGM] // WGM -v_sub_u32 v7, s[sgprNumWorkGroups1], v7 // WGM -v_cmpx_eq_u32 exec, v7, s[sgprWGM] // WGM -v_add_u32 v6, 1, v6 // WGM -s_mov_b64 exec, -1 // WGM -v_readfirstlane_b32 s80, v6 -s_mul_i32 s81, s[sgprWGM], s80 // quotient * non-magic divisor -s_sub_u32 s81, s[sgprNumWorkGroups1], s81 // WorkGroup1=remainder -s_cmp_eq_u32 s81, 0 // remainder == 0 ? -s_cmov_b32 s81, s[sgprWGM] // remainder = WGM if remainder == 0 -s_cmp_ge_u32 s82, s80 // blockId >= numFullBlocks ? -s_cselect_b32 s80, s81, s[sgprWGM] -v_cvt_f32_u32 v6, s80 // s[sgprWorkGroup0] = s83 / s80 -v_rcp_iflag_f32 v6, v6 // s[sgprWorkGroup0] = s83 / s80 -v_cvt_f32_u32 v7, s83 // s[sgprWorkGroup0] = s83 / s80 -v_mul_f32 v6, v6, v7 // s[sgprWorkGroup0] = s83 / s80 -v_cvt_u32_f32 v6, v6 // s[sgprWorkGroup0] = s83 / s80 -v_mul_u32_u24 v7, v6, s80 // s[sgprWorkGroup0] = s83 / s80 -v_sub_u32 v7, s83, v7 // s[sgprWorkGroup0] = s83 / s80 -v_cmpx_eq_u32 exec, v7, s80 // s[sgprWorkGroup0] = s83 / s80 -v_add_u32 v6, 1, v6 // s[sgprWorkGroup0] = s83 / s80 -v_mov_b32 v7, 0 // s[sgprWorkGroup1] = s83 % s80 -s_mov_b64 exec, -1 // s[sgprWorkGroup0] = s83 / s80 -v_readfirstlane_b32 s[sgprWorkGroup0], v6 -v_readfirstlane_b32 s[sgprWorkGroup1], v7 -s_mul_i32 s82, s82, s[sgprWGM] // blockId * WGM -s_add_u32 s[sgprWorkGroup1], s[sgprWorkGroup1], s82 // wg1 += blockId * WGM -label_WGM: - -/* global read addresses: tile offset assignment a */ -/* graTileAssignmentA = v0 */ - -/* global read addresses: tile offset assignment b */ -/* graTileAssignmentB = v2 */ - -/* global read addresses: unroll assignment a */ -/* v1 */ - -/* global read addresses: unroll assignment b */ -/* v3 */ - -/* global read addresses: other free assignments */ -/* s[sgprWorkGroup2] */ - -/* global read addresses: tile offsets a */ - -/* global read addresses: tile offsets b */ - -/* global read addresses: unroll offsets a */ - -/* global read addresses: unroll offsets b */ - -/* global read addresses: final offsets a */ -GLOBAL_OFFSET_A vgprGlobalReadOffsetA+0, 6 -s_mul_i32 s[sgprScalarGlobalReadOffsetA+0], s[sgprStrideA0I], 1 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+0], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetA+1], s[sgprStrideA0I], 2 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+1], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetA+2], s[sgprStrideA0I], 3 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+2], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetA+3], 1, 32 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+3], s[sgprScalarGlobalReadOffsetA+3], 0x1 // scalar offset *= bytes/element -s_add_u32 s[sgprScalarGlobalReadOffsetA+4], s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+3] -s_add_u32 s[sgprScalarGlobalReadOffsetA+5], s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+3] -s_add_u32 s[sgprScalarGlobalReadOffsetA+6], s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+3] - -/* global read addresses: final offsets b */ -GLOBAL_OFFSET_B vgprGlobalReadOffsetB+0, 3, 2, 6 // gROB_0_0_0_0 -s_mul_i32 s[sgprScalarGlobalReadOffsetB+0], s[sgprStrideB1J], 32 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+0], s[sgprScalarGlobalReadOffsetB+0], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetB+1], s[sgprStrideB1J], 64 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+1], s[sgprScalarGlobalReadOffsetB+1], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetB+2], s[sgprStrideB1J], 96 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+2], s[sgprScalarGlobalReadOffsetB+2], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetB+3], s[sgprStrideB1J], 128 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+3], s[sgprScalarGlobalReadOffsetB+3], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetB+4], s[sgprStrideB1J], 160 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+4], s[sgprScalarGlobalReadOffsetB+4], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetB+5], s[sgprStrideB1J], 192 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+5], s[sgprScalarGlobalReadOffsetB+5], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetB+6], s[sgprStrideB1J], 224 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetB+6], s[sgprScalarGlobalReadOffsetB+6], 0x1 // scalar offset *= bytes/element - -/* global read addresses: addresses a */ -/* max read offset = size[n] * stride[n-1] */ -s_mul_hi_u32 s83, s[sgprWorkGroup0], 256 // WorkGroup[01] * MT -s_mul_i32 s82, s[sgprWorkGroup0], 256 // WorkGroup[01] * MT -s_mul_hi_u32 s83, s82, s[sgprStrideA0I] // tlu=0, scaled tile-offset by stride -s_mul_i32 s82, s82, s[sgprStrideA0I] // tlu=0, scaled tile-offset by stride -s_mul_hi_u32 s81, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx -s_mul_i32 s80, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx -s_add_u32 s82, s82, s80 // accum GsuOffset term to tilestart -s_addc_u32 s83, s83, s81 // accum GsuOffset term to tilestart -s_mov_b32 s[sgprShadowLimitA+0], 1 // Init tensor size -s_mov_b32 s[sgprShadowLimitA+1], 0 // init tensor size -s_sub_u32 s80, s[sgprSizeL], 1 // (size-1) -s_mul_hi_u32 s81, constStrideAL, s80 // stride x (size-1) -s_mul_i32 s80, constStrideAL, s80 // stride x (size-1) -s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // sum tensor size -s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // sum tensor size -s_sub_u32 s80, s[sgprSizeI], 1 // (size-1) -s_mul_hi_u32 s81, s[sgprStrideA0I], s80 // stride x (size-1) -s_mul_i32 s80, s[sgprStrideA0I], s80 // stride x (size-1) -s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // sum tensor size -s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // sum tensor size -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s82 // sub tileStart -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s83 // sub tileStart -s_lshl_b64 s[sgprShadowLimitA:sgprShadowLimitA+1], s[sgprShadowLimitA:sgprShadowLimitA+1], 0x1 // Set limit to use bytes -s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], 16 // extend limit for pre-pad -s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], 0 // extend limit for pre-pad -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 -s_mul_hi_u32 s81, s[sgprStrideAK], s[sgprWorkGroup2] // Stride*WG -s_mul_i32 s80, s[sgprStrideAK], s[sgprWorkGroup2] // Stride*WG -s_add_u32 s82, s82, s80 // accum wg term to tilestart -s_addc_u32 s83, s83, s81 // accum wg term to tilestart -s_lshl_b64 s[82:83], s[82:83], 0x1 // tileStart *= BPE -s_add_u32 s[sgprSrdA+0], s[sgprAddressA+0], s82 // SRD base = Address+ tileStart0 -s_addc_u32 s[sgprSrdA+1], s[sgprAddressA+1], s83 // SRD base = Address+ tileStart1 -s_mov_b32 s[sgprSrdA+3], Srd127_96 // Set bits 127_96 in SRD - -/* global read addresses: addresses b */ -/* max read offset = size[n] * stride[n-1] */ -s_mul_hi_u32 s83, s[sgprWorkGroup1], 256 // WorkGroup[01] * MT -s_mul_i32 s82, s[sgprWorkGroup1], 256 // WorkGroup[01] * MT -s_mul_hi_u32 s83, s82, s[sgprStrideB1J] // tlu=0, scaled tile-offset by stride -s_mul_i32 s82, s82, s[sgprStrideB1J] // tlu=0, scaled tile-offset by stride -s_mul_hi_u32 s81, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx -s_mul_i32 s80, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx -s_add_u32 s82, s82, s80 // accum GsuOffset term to tilestart -s_addc_u32 s83, s83, s81 // accum GsuOffset term to tilestart -s_mov_b32 s[sgprShadowLimitB+0], 1 // Init tensor size -s_mov_b32 s[sgprShadowLimitB+1], 0 // init tensor size -s_sub_u32 s80, s[sgprSizeL], 1 // (size-1) -s_mul_hi_u32 s81, constStrideBL, s80 // stride x (size-1) -s_mul_i32 s80, constStrideBL, s80 // stride x (size-1) -s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // sum tensor size -s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // sum tensor size -s_sub_u32 s80, s[sgprSizeJ], 1 // (size-1) -s_mul_hi_u32 s81, s[sgprStrideB1J], s80 // stride x (size-1) -s_mul_i32 s80, s[sgprStrideB1J], s80 // stride x (size-1) -s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // sum tensor size -s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // sum tensor size -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s82 // sub tileStart -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s83 // sub tileStart -s_lshl_b64 s[sgprShadowLimitB:sgprShadowLimitB+1], s[sgprShadowLimitB:sgprShadowLimitB+1], 0x1 // Set limit to use bytes -s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], 16 // extend limit for pre-pad -s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], 0 // extend limit for pre-pad -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 -s_mul_hi_u32 s81, s[sgprStrideBK], s[sgprWorkGroup2] // Stride*WG -s_mul_i32 s80, s[sgprStrideBK], s[sgprWorkGroup2] // Stride*WG -s_add_u32 s82, s82, s80 // accum wg term to tilestart -s_addc_u32 s83, s83, s81 // accum wg term to tilestart -s_lshl_b64 s[82:83], s[82:83], 0x1 // tileStart *= BPE -s_add_u32 s[sgprSrdB+0], s[sgprAddressB+0], s82 // SRD base = Address+ tileStart0 -s_addc_u32 s[sgprSrdB+1], s[sgprAddressB+1], s83 // SRD base = Address+ tileStart1 -s_mov_b32 s[sgprSrdB+3], Srd127_96 // Set bits 127_96 in SRD -s_mul_i32 s80, s[sgprGSU], DepthU*BpeAGR -s_mov_b32 s[sgprGlobalReadIncsA+0], s80 // incrA (unrollIdx) - -/* global read addresses: increments b */ -s_mul_i32 s80, s[sgprGSU], DepthU*BpeBGR -s_mov_b32 s[sgprGlobalReadIncsB+0], s80 // incrB (unrollIdx) - -/* declare loop num iterations */ -s_lshr_b32 s[sgprLoopCounterL], s[sgprSizesSum+0], 6 // s[sgprLoopCounterL] = s[sgprSizesSum+0] / 64 -s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? -s_cbranch_scc1 label_GSU_1 // branch if GSU == 1 -v_cvt_f32_u32 v0, s[sgprGSU] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_rcp_iflag_f32 v0, v0 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_cvt_f32_u32 v1, s[sgprLoopCounterL] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_mul_f32 v0, v0, v1 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_cvt_u32_f32 v0, v0 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_mul_u32_u24 v1, v0, s[sgprGSU] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_sub_u32 v1, s[sgprLoopCounterL], v1 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_cmpx_eq_u32 exec, v1, s[sgprGSU] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_add_u32 v0, 1, v0 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_mov_b32 v1, 0 // s[sgprGSUSumIdx+1] = s[sgprLoopCounterL] % s[sgprGSU] -s_mov_b64 exec, -1 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] -v_readfirstlane_b32 s[sgprLoopCounterL], v0 -v_readfirstlane_b32 s[sgprGSUSumIdx+1], v1 -s_add_u32 s80, 1, s[sgprLoopCounterL] // tmp<-numIterMyWg+ -s_cmp_lt_u32 s[sgprGSUSumIdx], s[sgprGSUSumIdx+1] // gsuSumIdx < numIterPerWgRemainder -s_cmov_b32 s[sgprLoopCounterL], s80 // numIterMyWg++ if needed -label_GSU_1: -s_mov_b32 s[sgprOrigLoopCounter], s[sgprLoopCounterL] // copy loop counter -s_and_b32 s82, s[sgprStaggerU], 0x1f00 -s_lshr_b32 s82, s82, 0x8 -s_and_b32 s83, s[sgprStaggerU], 0xe000 -s_and_b32 s[sgprStaggerU], s[sgprStaggerU], 0xff -s_mov_b32 s80, s[sgprStaggerU] // init staggerU -label_beginStaggerUIter: -s_lshl_b32 s81, s80, s82 // shift by StaggerUStride -s_cmp_ge_u32 s[sgprOrigLoopCounter], s81 // loopCount >= current shift Count -s_cbranch_scc1 label_endStaggerUIter // jump to end -s_lshr_b32 s80, s80, 1 // step down to smaller stagger -s_branch label_beginStaggerUIter // jump to begin -label_endStaggerUIter: -s_sub_u32 s81, s80, 1 // staggerU mask -s_cmp_ge_u32 s80, 1 // if current staggerU >= 1 -s_cselect_b32 s[sgprStaggerUIter], s81, 0 // set Mask -s_cmp_eq_u32 s83, 0x0 -s_cbranch_scc1 label_StaggerUMapping_1 -s_mov_b32 s80, s[sgprWorkGroup0] -s_branch label_staggerInputEnd -label_StaggerUMapping_1: -s_cmp_eq_u32 s83, 0x2000 -s_cbranch_scc1 label_StaggerUMapping_2 -s_mov_b32 s80, s[sgprWorkGroup1] -s_branch label_staggerInputEnd -label_StaggerUMapping_2: -s_cmp_eq_u32 s83, 0x4000 -s_cbranch_scc1 label_StaggerUMapping_3 -s_mov_b32 s80, -0x1 -s_branch label_staggerInputEnd -label_StaggerUMapping_3: -s_cmp_eq_u32 s83, 0x6000 -s_cbranch_scc1 label_StaggerUMapping_4 -s_mul_i32 s81, s[sgprNumWorkGroups0], s[sgprWorkGroup1] -s_add_u32 s80, s80, s81 -s_add_u32 s80, s80, s[sgprWorkGroup0] -s_branch label_staggerInputEnd -label_StaggerUMapping_4: -s_cmp_eq_u32 s83, 0x8000 -s_cbranch_scc1 label_staggerInputEnd -s_mov_b32 s80, -0x1 -s_branch label_staggerInputEnd -label_staggerInputEnd: -s_and_b32 s[sgprStaggerUIter], s[sgprStaggerUIter], s80 // Compute actual stagger start for this tile -s_lshl_b32 s[sgprStaggerUIter], s[sgprStaggerUIter], s82 // shift by StaggerUStride - -/* SRDs += (StaggerUIter) * GlobalReadIncsA+0 */ -s_mul_hi_i32 s81, s[sgprStaggerUIter], s[sgprGlobalReadIncsA+0] // stagger byte offset -s_mul_i32 s80, s[sgprStaggerUIter], s[sgprGlobalReadIncsA+0] // stagger byte offset -s_mul_hi_i32 s[sgprWrapUA+1], s[sgprLoopCounterL], s[sgprGlobalReadIncsA+0] // Number of bytes accessed by the unroll loop -s_mul_i32 s[sgprWrapUA+0], s[sgprLoopCounterL], s[sgprGlobalReadIncsA+0] // Number of bytes accessed by the unroll loop -s_sub_u32 s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0], s[sgprWrapUA+0] // remove one iteration -s_subb_u32 s[sgprWrapUA+1], 0, s[sgprWrapUA+1] // remove one iteration -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 - -/* SRDs += (StaggerUIter) * GlobalReadIncsB+0 */ -s_mul_hi_i32 s81, s[sgprStaggerUIter], s[sgprGlobalReadIncsB+0] // stagger byte offset -s_mul_i32 s80, s[sgprStaggerUIter], s[sgprGlobalReadIncsB+0] // stagger byte offset -s_mul_hi_i32 s[sgprWrapUB+1], s[sgprLoopCounterL], s[sgprGlobalReadIncsB+0] // Number of bytes accessed by the unroll loop -s_mul_i32 s[sgprWrapUB+0], s[sgprLoopCounterL], s[sgprGlobalReadIncsB+0] // Number of bytes accessed by the unroll loop -s_sub_u32 s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0], s[sgprWrapUB+0] // remove one iteration -s_subb_u32 s[sgprWrapUB+1], 0, s[sgprWrapUB+1] // remove one iteration -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 -s_add_u32 s[sgprStaggerUIter], s[sgprStaggerUIter], 2 // Subtract (PGR-1); StaggerUIter now contains target iteration to wrap -/* local read addresses: init pointers a */ - -/* localReadInitPointers */ -/* local read addresses: init pointers b */ - -/* localReadInitPointers */ - -/* prefetch: global -> local */ -s_cmp_eq_u32 s[sgprLoopCounterL], 0 // at last iteration? -s_cbranch_scc1 label_ShadowInitStart // skip to ShadowInitStart iter b/c numIter==0 -buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 -buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 -buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 -buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 -buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 -buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 -buffer_load_dwordx4 v[vgprValuA_X0_I0+0:vgprValuA_X0_I0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0+0:vgprValuA_X2_I0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X0_I0+4:vgprValuA_X0_I0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0+4:vgprValuA_X2_I0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X0_I0+8:vgprValuA_X0_I0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_2_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0+8:vgprValuA_X2_I0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X0_I0+12:vgprValuA_X0_I0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_3_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0+12:vgprValuA_X2_I0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 - -/* global read inc A loopL */ -s_add_u32 s82, s[sgprLoopCounterL], 1 // remove pf(1) -s_cmp_eq_u32 s[sgprStaggerUIter], s82 // Is this wrapIter? (pf) -s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? -s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 - -/* global read inc B loopL */ -s_add_u32 s82, s[sgprLoopCounterL], 1 // remove pf(1) -s_cmp_eq_u32 s[sgprStaggerUIter], s82 // Is this wrapIter? (pf) -s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? -s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 - -/******************************************/ -/* End setupNewTile */ -/******************************************/ -label_ShadowInitStart: -s_mov_b32 s[sgprSrdD+0], s[sgprAddressD+0] // init SRD base address (lower) -s_mov_b32 s[sgprSrdD+1], s[sgprAddressD+1] // init SRD base address (upper) + other fields -s_mov_b32 s[sgprSrdD+2], 0x80000000 -s_mov_b32 s[sgprSrdD+3], Srd127_96 // Set bits 127_96 in post-loop SRD - -s_mov_b32 s[sgprSrdC+0], s[sgprAddressC+0] // init SRD base address (lower) -s_mov_b32 s[sgprSrdC+1], s[sgprAddressC+1] // init SRD base address (upper) + other fields -s_mov_b32 s[sgprSrdC+2], 0x80000000 -s_mov_b32 s[sgprSrdC+3], Srd127_96 // Set bits 127_96 in post-loop SRD - - -s_mul_i32 s82, MT1, s[sgprWorkGroup1] // <- wg1*MT1 -s_mul_hi_u32 s81, s82, s[sgprStrideC1J] // ScaleC s82 by Stride -s_mul_i32 s80, s82, s[sgprStrideC1J] // ScaleC s82 by Stride -s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeC] // scale by bpe -s_add_u32 s[sgprSrdC+0], s[sgprAddressC+0], s80 // add lo to SRD -s_addc_u32 s[sgprSrdC+1], s[sgprAddressC+1], s81 // add hi to SRD -s_mul_hi_u32 s81, s82, s[sgprStrideD1J] // ScaleD s82 by Stride -s_mul_i32 s80, s82, s[sgprStrideD1J] // ScaleD s82 by Stride -s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeD] // scale by bpe -s_add_u32 s[sgprSrdD+0], s[sgprAddressD+0], s80 // add lo to SRD -s_addc_u32 s[sgprSrdD+1], s[sgprAddressD+1], s81 // add hi to SRD - -s_mul_hi_u32 s81, s[sgprWorkGroup2], s[sgprStrideCK] // ScaleC s[sgprWorkGroup2] by Stride -s_mul_i32 s80, s[sgprWorkGroup2], s[sgprStrideCK] // ScaleC s[sgprWorkGroup2] by Stride -s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeC] // scale by bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s80 // add lo to SRD -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], s81 // add hi to SRD -s_mul_hi_u32 s81, s[sgprWorkGroup2], s[sgprStrideDK] // ScaleD s[sgprWorkGroup2] by Stride -s_mul_i32 s80, s[sgprWorkGroup2], s[sgprStrideDK] // ScaleD s[sgprWorkGroup2] by Stride -s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeD] // scale by bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s80 // add lo to SRD -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], s81 // add hi to SRD - -s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? -s_cbranch_scc1 label_GSU_2 // branch if GSU == 1 -// GSU Output Buffer offset: Free0 + (Free1-1)*StrideC1J + (Free2-1)*StrideCK * GSUIdx * bpe%s -s_mul_hi_u32 s81, s[sgprSizesFree+0], s[sgprGSUSumIdx] // Free0 -s_mul_i32 s80, s[sgprSizesFree+0], s[sgprGSUSumIdx] // Free0 -s_sub_u32 s82, s[sgprSizesFree+1], 1 // Free1 -s_mul_i32 s82, s82, s[sgprGSUSumIdx] // Free1 -s_mul_hi_u32 s83, s82, s[sgprStrideC1J] // Free1 -s_mul_i32 s82, s82, s[sgprStrideC1J] // Free1 -s_add_u32 s80, s80, s82 // Free1 -s_addc_u32 s81, s81, s83 // Free1 -s_sub_u32 s82, s[sgprSizesFree+2], 1 // Free2 -s_mul_i32 s82, s82, s[sgprGSUSumIdx] // Free2 -s_mul_hi_u32 s83, s82, s[sgprStrideCK] // Free2 -s_mul_i32 s82, s82, s[sgprStrideCK] // Free2 -s_add_u32 s80, s80, s82 // Free2 -s_addc_u32 s81, s81, s83 // Free2 -s_lshl_b64 s[80:81], s[80:81], 2 // scale by bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s80 // add lo GSU offset to SRD -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], s81 // add hi GSU offset to SRD -label_GSU_2: -.set sgprGSULog2BpeC, UNDEF - -/* initC: remove ValuC vgpr buffer [0...0) from pool */ - -/* initC: remove acc vgpr buffer [0...256) from pool */ - -/* initC: remove ValuA/B vgpr buffer [0...160) from pool */ -v_accvgpr_write acc0, 0x0 // initC -v_accvgpr_write acc1, 0x0 // initC -v_accvgpr_write acc2, 0x0 // initC -v_accvgpr_write acc3, 0x0 // initC -v_accvgpr_write acc4, 0x0 // initC -v_accvgpr_write acc5, 0x0 // initC -v_accvgpr_write acc6, 0x0 // initC -v_accvgpr_write acc7, 0x0 // initC -v_accvgpr_write acc8, 0x0 // initC -v_accvgpr_write acc9, 0x0 // initC -v_accvgpr_write acc10, 0x0 // initC -v_accvgpr_write acc11, 0x0 // initC -v_accvgpr_write acc12, 0x0 // initC -v_accvgpr_write acc13, 0x0 // initC -v_accvgpr_write acc14, 0x0 // initC -v_accvgpr_write acc15, 0x0 // initC -v_accvgpr_write acc16, 0x0 // initC -v_accvgpr_write acc17, 0x0 // initC -v_accvgpr_write acc18, 0x0 // initC -v_accvgpr_write acc19, 0x0 // initC -v_accvgpr_write acc20, 0x0 // initC -v_accvgpr_write acc21, 0x0 // initC -v_accvgpr_write acc22, 0x0 // initC -v_accvgpr_write acc23, 0x0 // initC -v_accvgpr_write acc24, 0x0 // initC -v_accvgpr_write acc25, 0x0 // initC -v_accvgpr_write acc26, 0x0 // initC -v_accvgpr_write acc27, 0x0 // initC -v_accvgpr_write acc28, 0x0 // initC -v_accvgpr_write acc29, 0x0 // initC -v_accvgpr_write acc30, 0x0 // initC -v_accvgpr_write acc31, 0x0 // initC -v_accvgpr_write acc32, 0x0 // initC -v_accvgpr_write acc33, 0x0 // initC -v_accvgpr_write acc34, 0x0 // initC -v_accvgpr_write acc35, 0x0 // initC -v_accvgpr_write acc36, 0x0 // initC -v_accvgpr_write acc37, 0x0 // initC -v_accvgpr_write acc38, 0x0 // initC -v_accvgpr_write acc39, 0x0 // initC -v_accvgpr_write acc40, 0x0 // initC -v_accvgpr_write acc41, 0x0 // initC -v_accvgpr_write acc42, 0x0 // initC -v_accvgpr_write acc43, 0x0 // initC -v_accvgpr_write acc44, 0x0 // initC -v_accvgpr_write acc45, 0x0 // initC -v_accvgpr_write acc46, 0x0 // initC -v_accvgpr_write acc47, 0x0 // initC -v_accvgpr_write acc48, 0x0 // initC -v_accvgpr_write acc49, 0x0 // initC -v_accvgpr_write acc50, 0x0 // initC -v_accvgpr_write acc51, 0x0 // initC -v_accvgpr_write acc52, 0x0 // initC -v_accvgpr_write acc53, 0x0 // initC -v_accvgpr_write acc54, 0x0 // initC -v_accvgpr_write acc55, 0x0 // initC -v_accvgpr_write acc56, 0x0 // initC -v_accvgpr_write acc57, 0x0 // initC -v_accvgpr_write acc58, 0x0 // initC -v_accvgpr_write acc59, 0x0 // initC -v_accvgpr_write acc60, 0x0 // initC -v_accvgpr_write acc61, 0x0 // initC -v_accvgpr_write acc62, 0x0 // initC -v_accvgpr_write acc63, 0x0 // initC -v_accvgpr_write acc64, 0x0 // initC -v_accvgpr_write acc65, 0x0 // initC -v_accvgpr_write acc66, 0x0 // initC -v_accvgpr_write acc67, 0x0 // initC -v_accvgpr_write acc68, 0x0 // initC -v_accvgpr_write acc69, 0x0 // initC -v_accvgpr_write acc70, 0x0 // initC -v_accvgpr_write acc71, 0x0 // initC -v_accvgpr_write acc72, 0x0 // initC -v_accvgpr_write acc73, 0x0 // initC -v_accvgpr_write acc74, 0x0 // initC -v_accvgpr_write acc75, 0x0 // initC -v_accvgpr_write acc76, 0x0 // initC -v_accvgpr_write acc77, 0x0 // initC -v_accvgpr_write acc78, 0x0 // initC -v_accvgpr_write acc79, 0x0 // initC -v_accvgpr_write acc80, 0x0 // initC -v_accvgpr_write acc81, 0x0 // initC -v_accvgpr_write acc82, 0x0 // initC -v_accvgpr_write acc83, 0x0 // initC -v_accvgpr_write acc84, 0x0 // initC -v_accvgpr_write acc85, 0x0 // initC -v_accvgpr_write acc86, 0x0 // initC -v_accvgpr_write acc87, 0x0 // initC -v_accvgpr_write acc88, 0x0 // initC -v_accvgpr_write acc89, 0x0 // initC -v_accvgpr_write acc90, 0x0 // initC -v_accvgpr_write acc91, 0x0 // initC -v_accvgpr_write acc92, 0x0 // initC -v_accvgpr_write acc93, 0x0 // initC -v_accvgpr_write acc94, 0x0 // initC -v_accvgpr_write acc95, 0x0 // initC -v_accvgpr_write acc96, 0x0 // initC -v_accvgpr_write acc97, 0x0 // initC -v_accvgpr_write acc98, 0x0 // initC -v_accvgpr_write acc99, 0x0 // initC -v_accvgpr_write acc100, 0x0 // initC -v_accvgpr_write acc101, 0x0 // initC -v_accvgpr_write acc102, 0x0 // initC -v_accvgpr_write acc103, 0x0 // initC -v_accvgpr_write acc104, 0x0 // initC -v_accvgpr_write acc105, 0x0 // initC -v_accvgpr_write acc106, 0x0 // initC -v_accvgpr_write acc107, 0x0 // initC -v_accvgpr_write acc108, 0x0 // initC -v_accvgpr_write acc109, 0x0 // initC -v_accvgpr_write acc110, 0x0 // initC -v_accvgpr_write acc111, 0x0 // initC -v_accvgpr_write acc112, 0x0 // initC -v_accvgpr_write acc113, 0x0 // initC -v_accvgpr_write acc114, 0x0 // initC -v_accvgpr_write acc115, 0x0 // initC -v_accvgpr_write acc116, 0x0 // initC -v_accvgpr_write acc117, 0x0 // initC -v_accvgpr_write acc118, 0x0 // initC -v_accvgpr_write acc119, 0x0 // initC -v_accvgpr_write acc120, 0x0 // initC -v_accvgpr_write acc121, 0x0 // initC -v_accvgpr_write acc122, 0x0 // initC -v_accvgpr_write acc123, 0x0 // initC -v_accvgpr_write acc124, 0x0 // initC -v_accvgpr_write acc125, 0x0 // initC -v_accvgpr_write acc126, 0x0 // initC -v_accvgpr_write acc127, 0x0 // initC -v_accvgpr_write acc128, 0x0 // initC -v_accvgpr_write acc129, 0x0 // initC -v_accvgpr_write acc130, 0x0 // initC -v_accvgpr_write acc131, 0x0 // initC -v_accvgpr_write acc132, 0x0 // initC -v_accvgpr_write acc133, 0x0 // initC -v_accvgpr_write acc134, 0x0 // initC -v_accvgpr_write acc135, 0x0 // initC -v_accvgpr_write acc136, 0x0 // initC -v_accvgpr_write acc137, 0x0 // initC -v_accvgpr_write acc138, 0x0 // initC -v_accvgpr_write acc139, 0x0 // initC -v_accvgpr_write acc140, 0x0 // initC -v_accvgpr_write acc141, 0x0 // initC -v_accvgpr_write acc142, 0x0 // initC -v_accvgpr_write acc143, 0x0 // initC -v_accvgpr_write acc144, 0x0 // initC -v_accvgpr_write acc145, 0x0 // initC -v_accvgpr_write acc146, 0x0 // initC -v_accvgpr_write acc147, 0x0 // initC -v_accvgpr_write acc148, 0x0 // initC -v_accvgpr_write acc149, 0x0 // initC -v_accvgpr_write acc150, 0x0 // initC -v_accvgpr_write acc151, 0x0 // initC -v_accvgpr_write acc152, 0x0 // initC -v_accvgpr_write acc153, 0x0 // initC -v_accvgpr_write acc154, 0x0 // initC -v_accvgpr_write acc155, 0x0 // initC -v_accvgpr_write acc156, 0x0 // initC -v_accvgpr_write acc157, 0x0 // initC -v_accvgpr_write acc158, 0x0 // initC -v_accvgpr_write acc159, 0x0 // initC -v_accvgpr_write acc160, 0x0 // initC -v_accvgpr_write acc161, 0x0 // initC -v_accvgpr_write acc162, 0x0 // initC -v_accvgpr_write acc163, 0x0 // initC -v_accvgpr_write acc164, 0x0 // initC -v_accvgpr_write acc165, 0x0 // initC -v_accvgpr_write acc166, 0x0 // initC -v_accvgpr_write acc167, 0x0 // initC -v_accvgpr_write acc168, 0x0 // initC -v_accvgpr_write acc169, 0x0 // initC -v_accvgpr_write acc170, 0x0 // initC -v_accvgpr_write acc171, 0x0 // initC -v_accvgpr_write acc172, 0x0 // initC -v_accvgpr_write acc173, 0x0 // initC -v_accvgpr_write acc174, 0x0 // initC -v_accvgpr_write acc175, 0x0 // initC -v_accvgpr_write acc176, 0x0 // initC -v_accvgpr_write acc177, 0x0 // initC -v_accvgpr_write acc178, 0x0 // initC -v_accvgpr_write acc179, 0x0 // initC -v_accvgpr_write acc180, 0x0 // initC -v_accvgpr_write acc181, 0x0 // initC -v_accvgpr_write acc182, 0x0 // initC -v_accvgpr_write acc183, 0x0 // initC -v_accvgpr_write acc184, 0x0 // initC -v_accvgpr_write acc185, 0x0 // initC -v_accvgpr_write acc186, 0x0 // initC -v_accvgpr_write acc187, 0x0 // initC -v_accvgpr_write acc188, 0x0 // initC -v_accvgpr_write acc189, 0x0 // initC -v_accvgpr_write acc190, 0x0 // initC -v_accvgpr_write acc191, 0x0 // initC -v_accvgpr_write acc192, 0x0 // initC -v_accvgpr_write acc193, 0x0 // initC -v_accvgpr_write acc194, 0x0 // initC -v_accvgpr_write acc195, 0x0 // initC -v_accvgpr_write acc196, 0x0 // initC -v_accvgpr_write acc197, 0x0 // initC -v_accvgpr_write acc198, 0x0 // initC -v_accvgpr_write acc199, 0x0 // initC -v_accvgpr_write acc200, 0x0 // initC -v_accvgpr_write acc201, 0x0 // initC -v_accvgpr_write acc202, 0x0 // initC -v_accvgpr_write acc203, 0x0 // initC -v_accvgpr_write acc204, 0x0 // initC -v_accvgpr_write acc205, 0x0 // initC -v_accvgpr_write acc206, 0x0 // initC -v_accvgpr_write acc207, 0x0 // initC -v_accvgpr_write acc208, 0x0 // initC -v_accvgpr_write acc209, 0x0 // initC -v_accvgpr_write acc210, 0x0 // initC -v_accvgpr_write acc211, 0x0 // initC -v_accvgpr_write acc212, 0x0 // initC -v_accvgpr_write acc213, 0x0 // initC -v_accvgpr_write acc214, 0x0 // initC -v_accvgpr_write acc215, 0x0 // initC -v_accvgpr_write acc216, 0x0 // initC -v_accvgpr_write acc217, 0x0 // initC -v_accvgpr_write acc218, 0x0 // initC -v_accvgpr_write acc219, 0x0 // initC -v_accvgpr_write acc220, 0x0 // initC -v_accvgpr_write acc221, 0x0 // initC -v_accvgpr_write acc222, 0x0 // initC -v_accvgpr_write acc223, 0x0 // initC -v_accvgpr_write acc224, 0x0 // initC -v_accvgpr_write acc225, 0x0 // initC -v_accvgpr_write acc226, 0x0 // initC -v_accvgpr_write acc227, 0x0 // initC -v_accvgpr_write acc228, 0x0 // initC -v_accvgpr_write acc229, 0x0 // initC -v_accvgpr_write acc230, 0x0 // initC -v_accvgpr_write acc231, 0x0 // initC -v_accvgpr_write acc232, 0x0 // initC -v_accvgpr_write acc233, 0x0 // initC -v_accvgpr_write acc234, 0x0 // initC -v_accvgpr_write acc235, 0x0 // initC -v_accvgpr_write acc236, 0x0 // initC -v_accvgpr_write acc237, 0x0 // initC -v_accvgpr_write acc238, 0x0 // initC -v_accvgpr_write acc239, 0x0 // initC -v_accvgpr_write acc240, 0x0 // initC -v_accvgpr_write acc241, 0x0 // initC -v_accvgpr_write acc242, 0x0 // initC -v_accvgpr_write acc243, 0x0 // initC -v_accvgpr_write acc244, 0x0 // initC -v_accvgpr_write acc245, 0x0 // initC -v_accvgpr_write acc246, 0x0 // initC -v_accvgpr_write acc247, 0x0 // initC -v_accvgpr_write acc248, 0x0 // initC -v_accvgpr_write acc249, 0x0 // initC -v_accvgpr_write acc250, 0x0 // initC -v_accvgpr_write acc251, 0x0 // initC -v_accvgpr_write acc252, 0x0 // initC -v_accvgpr_write acc253, 0x0 // initC -v_accvgpr_write acc254, 0x0 // initC -v_accvgpr_write acc255, 0x0 // initC -s_cmp_eq_u32 s[sgprLoopCounterL], 0 // at last iteration? - -/* after InitC, skip to end of prefetch last iter if numIter==0 */ -s_cbranch_scc0 label_NoBranch_5L94Q2PNAOS4OEC5_0 // Only branch on scc1 -s_getpc_b64 s[80:81] // addr of next instr -s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset -s_add_u32 s80, s80, s82 // add target branch offset -s_addc_u32 s81, s81, 0 // add high and carry -s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd -label_NoBranch_5L94Q2PNAOS4OEC5_0: -s_waitcnt vmcnt(8) -/* local write a */ - -/* local write b */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 - -/* local write swap a */ - -/* local write swap b */ -s_cmp_eq_u32 s[sgprLoopCounterL], 0x1 // PGR=2 but only 1 loop -s_cbranch_scc1 label_skipPGR2_0 // PGR=2 but only 1 loop -buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 -buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 -buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 -buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 -buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 -buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 - -label_skipPGR2_0: -s_waitcnt lgkmcnt(0) // 0prefetch wait for local write -// Skip force waitcnt0 -s_barrier - -/* local read prefetch a */ - -/* local read prefetch b */ -ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 - -/* local read inc a */ -/* N/A, lro->32 */ -/* self.localReadDoCntA 1 self.localReadDoCntB 1 */ - -/* local read inc b */ -/* N/A, lro->32 */ -/* self.localReadDoCntA 1 self.localReadDoCntB 1 */ - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 - -/******************************************/ -/* Unrolled Loop(s) - Begin */ -/******************************************/ -label_openLoopL: -s_cmp_eq_u32 s[sgprLoopCounterL], 0x1 // LoopCounterL < EndCounter -s_cbranch_scc1 label_LoopEndL_odd_NoLoadLoop -s_cmp_le_u32 s[sgprLoopCounterL], 0x2 // LoopCounterL < EndCounter -s_cbranch_scc1 label_LoopEndL_even // do not enter LoopL -label_LoopBeginL: - -/******************************************/ -/* Unrolled Loop 1/2 - Begin */ -/******************************************/ - -s_waitcnt vmcnt(8) - -/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ - -/* iter 0 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:0 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:1 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+0:vgprValuA_X0_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+0:vgprValuA_X2_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc B loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:2 */ -ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:3 */ -s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:4 */ -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:5 */ -ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:6 */ -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:7 */ -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:8 */ -ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:9 */ -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:10 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:11 */ -ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:12 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:13 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:14 */ -ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:15 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:16 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:17 */ -ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:18 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:19 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:20 */ -ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:21 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:22 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:23 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:24 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:25 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:26 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:27 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:28 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:29 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -/* mfmaIndex:30 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:31 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:32 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+4:vgprValuA_X0_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+4:vgprValuA_X2_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:33 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:34 */ -/* schedule remaining localreads for 1LDSB */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:35 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:36 */ -ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:37 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:38 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:39 */ -ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:40 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:41 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:42 */ -ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:43 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:44 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:45 */ -ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:46 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:47 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:48 */ -ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:49 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:50 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:51 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:52 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:53 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:54 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:55 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:56 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:57 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:58 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:59 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:60 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:61 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:62 */ -/* 1 LDS buffer: read-sync-write */ -s_waitcnt lgkmcnt(0) -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:63 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 1 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:64 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:65 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+8:vgprValuA_X0_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_2_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+8:vgprValuA_X2_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:66 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:67 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:68 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:69 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:70 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:71 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:72 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:73 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:74 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:75 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:76 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:77 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:78 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:79 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:80 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:81 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:82 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:83 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:84 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:85 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:86 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:87 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:88 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:89 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:90 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:91 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:92 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:93 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:94 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:95 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:96 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+12:vgprValuA_X0_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_3_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+12:vgprValuA_X2_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc A loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:97 */ -s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:98 */ -s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:99 */ -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:100 */ -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:101 */ -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:102 */ -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:103 */ -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:104 */ -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:105 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:106 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:107 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:108 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:109 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:110 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:111 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:112 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:113 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:114 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:115 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:116 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:117 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:118 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:119 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:120 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:121 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:122 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:123 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:124 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:125 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:126 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:127 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:128 */ -buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:129 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:130 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:131 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:132 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:133 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:134 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:135 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:136 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:137 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:138 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:139 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:140 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:141 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:142 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:143 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:144 */ -buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:145 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:146 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:147 */ -/* sched write - iter 2 writesPerItem=1 */ -/* sched write - iter 2 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:148 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:149 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:150 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:151 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:152 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:153 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:154 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:155 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:156 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:157 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:158 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:159 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:160 */ -buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:161 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:162 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:163 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:164 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:165 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:166 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:167 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:168 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:169 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:170 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:171 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:172 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:173 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:174 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:175 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:176 */ -buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:177 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:178 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:179 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:180 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:181 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:182 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:183 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:184 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:185 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:186 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:187 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:188 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:189 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:190 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:191 */ - -/* local read swap offsets a */ - -/* local read swap offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ - -/* iter 3 (swap and reset local write pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:192 */ -buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:193 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:194 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:195 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:196 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:197 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:198 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:199 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:200 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:201 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:202 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:203 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:204 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:205 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:206 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:207 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:208 */ -buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:209 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:210 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:211 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:212 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:213 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:214 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:215 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:216 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:217 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:218 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:219 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:220 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:221 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:222 */ - -/* local write swap offsets a */ - -/* local write swap offsets b */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:223 */ -ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:224 */ -buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:225 */ -ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:226 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:227 */ -ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:228 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:229 */ -ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:230 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:231 */ -ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:232 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:233 */ -ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:234 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:235 */ -ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:236 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:237 */ -ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:238 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:239 */ -ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:240 */ -buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:241 */ -ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:242 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:243 */ -ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:244 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:245 */ -ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:246 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:247 */ -ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:248 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:249 */ -ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:250 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:251 */ -ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:252 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:253 */ -ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:254 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:255 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=1 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/******************************************/ -/* Unrolled Loop - End */ -/******************************************/ - -/* closeLoop loopL finalLoop=1 tailLoop=0 */ -s_sub_u32 s[sgprLoopCounterL], s[sgprLoopCounterL], 1 // dec counterL -s_cmp_eq_i32 s[sgprLoopCounterL], 0x2 // counterL==1 -s_cbranch_scc1 label_LoopEndL_odd // to End - -/******************************************/ -/* Unrolled Loop 2/2 - Begin */ -/******************************************/ - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 - -s_waitcnt vmcnt(8) - -/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ - -/* iter 0 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:0 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:1 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+0:vgprValuA_X0_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+0:vgprValuA_X2_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc B loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:2 */ -ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:3 */ -s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:4 */ -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:5 */ -ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:6 */ -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:7 */ -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:8 */ -ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:9 */ -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:10 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:11 */ -ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:12 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:13 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:14 */ -ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:15 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:16 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:17 */ -ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:18 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:19 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:20 */ -ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:21 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:22 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:23 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:24 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:25 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:26 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:27 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:28 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:29 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:30 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:31 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:32 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+4:vgprValuA_X0_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+4:vgprValuA_X2_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:33 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:34 */ -/* schedule remaining localreads for 1LDSB */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:35 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:36 */ -ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:37 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:38 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:39 */ -ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:40 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:41 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:42 */ -ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:43 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:44 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:45 */ -ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:46 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:47 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:48 */ -ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:49 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:50 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:51 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:52 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:53 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:54 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:55 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:56 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:57 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:58 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:59 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:60 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:61 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:62 */ -/* 1 LDS buffer: read-sync-write */ -s_waitcnt lgkmcnt(0) -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:63 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 1 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:64 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:65 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+8:vgprValuA_X0_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_2_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+8:vgprValuA_X2_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:66 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:67 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:68 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:69 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:70 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:71 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:72 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:73 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:74 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:75 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:76 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:77 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:78 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:79 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:80 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:81 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:82 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:83 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:84 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:85 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:86 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:87 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:88 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:89 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:90 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:91 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:92 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:93 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:94 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:95 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:96 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+12:vgprValuA_X0_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_3_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+12:vgprValuA_X2_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc A loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:97 */ -s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:98 */ -s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:99 */ -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:100 */ -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:101 */ -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:102 */ -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:103 */ -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:104 */ -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:105 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:106 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:107 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:108 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:109 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:110 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:111 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:112 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:113 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:114 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:115 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:116 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:117 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:118 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:119 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:120 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:121 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:122 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:123 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:124 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:125 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:126 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:127 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:128 */ -buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:129 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:130 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:131 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:132 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:133 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:134 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:135 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:136 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:137 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:138 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:139 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:140 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:141 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:142 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:143 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:144 */ -buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:145 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:146 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:147 */ -/* sched write - iter 2 writesPerItem=1 */ -/* sched write - iter 2 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:148 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:149 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:150 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:151 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:152 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:153 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:154 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:155 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:156 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:157 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:158 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:159 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:160 */ -buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:161 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:162 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:163 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:164 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:165 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:166 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:167 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:168 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:169 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:170 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:171 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:172 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:173 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:174 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:175 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:176 */ -buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:177 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:178 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:179 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:180 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:181 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:182 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:183 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:184 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:185 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:186 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:187 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:188 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:189 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:190 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:191 */ - -/* local read swap offsets a */ - -/* local read swap offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ - -/* iter 3 (swap and reset local write pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:192 */ -buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:193 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:194 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:195 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:196 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:197 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:198 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:199 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:200 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:201 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:202 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:203 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:204 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:205 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:206 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:207 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:208 */ -buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:209 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:210 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:211 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:212 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:213 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:214 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:215 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:216 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:217 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:218 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:219 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:220 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:221 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:222 */ -/* local write swap offsets a */ - -/* local write swap offsets b */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:223 */ -ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:224 */ -buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:225 */ -ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:226 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:227 */ -ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:228 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:229 */ -ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:230 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:231 */ -ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:232 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:233 */ -ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:234 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:235 */ -ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:236 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:237 */ -ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:238 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:239 */ -ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:240 */ -buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:241 */ -ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:242 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:243 */ -ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:244 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:245 */ -ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:246 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:247 */ -ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:248 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:249 */ -ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:250 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:251 */ -ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:252 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:253 */ -ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:254 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:255 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=1 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 -/******************************************/ -/* Unrolled Loop - End */ -/******************************************/ - -/* closeLoop loopL finalLoop=1 tailLoop=0 */ -s_sub_u32 s[sgprLoopCounterL], s[sgprLoopCounterL], 1 // dec counterL -s_cmp_eq_i32 s[sgprLoopCounterL], 0x2 // counterL==2 -s_cbranch_scc0 label_LoopBeginL // restart LoopL -label_LoopEndL_even: - -/* Before NLL: Check VGPR.checkin for INT8 LW */ - -/******************************************/ -/* Ord. NoGlobalLoadLoop - Begin */ -/******************************************/ -s_waitcnt vmcnt(8) -/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ - -/* iter 0 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:0 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:1 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+0:vgprValuA_X0_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+0:vgprValuA_X2_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc B loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:2 */ -ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:3 */ -s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:4 */ -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:5 */ -ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:6 */ -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:7 */ -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:8 */ -ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:9 */ -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:10 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:11 */ -ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:12 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:13 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:14 */ -ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:15 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:16 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:17 */ -ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:18 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:19 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:20 */ -ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:21 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:22 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:23 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:24 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:25 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:26 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:27 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:28 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:29 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -/* mfmaIndex:30 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:31 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:32 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+4:vgprValuA_X0_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+4:vgprValuA_X2_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:33 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:34 */ -/* schedule remaining localreads for 1LDSB */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:35 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:36 */ -ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:37 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:38 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:39 */ -ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:40 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:41 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:42 */ -ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:43 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:44 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:45 */ -ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:46 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:47 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:48 */ -ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:49 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:50 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:51 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:52 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:53 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:54 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:55 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:56 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:57 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:58 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:59 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:60 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:61 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:62 */ -/* 1 LDS buffer: read-sync-write */ -s_waitcnt lgkmcnt(0) -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:63 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 1 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:64 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:65 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+8:vgprValuA_X0_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+8:vgprValuA_X2_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:66 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:67 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:68 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:69 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:70 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:71 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:72 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:73 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:74 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:75 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:76 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:77 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:78 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:79 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:80 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:81 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:82 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:83 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:84 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:85 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:86 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:87 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:88 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:89 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:90 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:91 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:92 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:93 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:94 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:95 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:96 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_1+12:vgprValuA_X0_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_1+12:vgprValuA_X2_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc A loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:97 */ -s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:98 */ -s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:99 */ -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:100 */ -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:101 */ -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:102 */ -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:103 */ -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:104 */ -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:105 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:106 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:107 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:108 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:109 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:110 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:111 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:112 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:113 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:114 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:115 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:116 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:117 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:118 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:119 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:120 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:121 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:122 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:123 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:124 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:125 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:126 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:127 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:128 */ -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:129 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:130 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:131 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:132 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:133 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:134 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:135 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:136 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:137 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:138 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:139 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:140 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:141 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:142 */ -s_waitcnt vmcnt(10) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:143 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:144 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:145 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:146 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:147 */ -/* sched write - iter 2 writesPerItem=1 */ -/* sched write - iter 2 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:148 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:149 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:150 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:151 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:152 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:153 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:154 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:155 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:156 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:157 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:158 */ -s_waitcnt vmcnt(9) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:159 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:160 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:161 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:162 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:163 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:164 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:165 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:166 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:167 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:168 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:169 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:170 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:171 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:172 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:173 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:174 */ -s_waitcnt vmcnt(8) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:175 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:176 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:177 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:178 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:179 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:180 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:181 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:182 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:183 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:184 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:185 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:186 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:187 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:188 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:189 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:190 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:191 */ - -/* local read swap offsets a */ - -/* local read swap offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ - -/* iter 3 (swap and reset local write pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:192 */ -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:193 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:194 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:195 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:196 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:197 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:198 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:199 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:200 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:201 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:202 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:203 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:204 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:205 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:206 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:207 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:208 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:209 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:210 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:211 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:212 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:213 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:214 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:215 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:216 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:217 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:218 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:219 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:220 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:221 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:222 */ - -/* local write swap offsets a */ - -/* local write swap offsets b */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:223 */ -ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:224 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:225 */ -ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:226 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:227 */ -ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:228 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:229 */ -ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:230 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:231 */ -ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:232 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:233 */ -ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:234 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:235 */ -ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:236 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:237 */ -ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:238 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:239 */ -ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:240 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:241 */ -ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:242 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:243 */ -ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:244 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:245 */ -ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:246 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:247 */ -ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:248 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:249 */ -ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:250 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:251 */ -ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:252 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:253 */ -ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:254 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:255 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=1 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 - -/******************************************/ -/* Ord. NoLoadLoop - Begin */ -/******************************************/ -s_waitcnt vmcnt(0) - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 - -/* iter 0 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:0 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:1 */ -ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:2 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:3 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:4 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:5 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:6 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:7 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:8 */ -ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:9 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:10 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:11 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:12 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:13 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:14 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:15 */ -ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:16 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:17 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:18 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:19 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:20 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:21 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:22 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:23 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:24 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:25 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:26 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:27 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:28 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:29 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:30 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:31 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:32 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:33 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:34 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:35 */ -/* sched write - iter 0 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:36 */ -ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:37 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:38 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:39 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:40 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:41 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:42 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:43 */ -ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:44 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:45 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:46 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:47 */ -/* sched write - iter 0 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:48 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:49 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:50 */ -ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:51 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:52 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:53 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:54 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:55 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:56 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:57 */ -ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:58 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:59 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:60 */ -/* sched write - iter 0 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:61 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:62 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:63 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 1 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:64 */ -ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:65 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:66 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:67 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:68 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:69 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:70 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:71 */ -ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:72 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:73 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:74 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:75 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:76 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:77 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:78 */ -ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:79 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:80 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:81 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:82 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:83 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:84 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:85 */ -ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:86 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:87 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:88 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:89 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:90 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:91 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:92 */ -ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:93 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:94 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:95 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:96 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:97 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:98 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:99 */ -ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:100 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:101 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:102 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:103 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:104 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:105 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:106 */ -ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:107 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:108 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:109 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:110 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:111 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:112 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:113 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:114 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:115 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:116 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:117 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:118 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:119 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:120 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:121 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:122 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:123 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:124 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:125 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:126 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:127 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:128 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:129 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:130 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:131 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:132 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:133 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:134 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:135 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:136 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:137 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:138 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:139 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:140 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:141 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:142 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:143 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:144 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:145 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:146 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:147 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:148 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:149 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:150 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:151 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:152 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:153 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:154 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:155 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:156 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:157 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:158 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:159 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:160 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:161 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:162 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:163 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:164 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:165 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:166 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:167 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:168 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:169 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:170 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:171 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:172 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:173 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:174 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:175 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:176 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:177 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:178 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:179 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:180 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:181 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:182 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:183 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:184 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:185 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:186 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:187 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:188 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:189 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:190 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:191 */ - -/* local read swap offsets a */ - -/* local read swap offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ - -/* iter 3 (swap and reset local write pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:192 */ -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:193 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:194 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:195 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:196 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:197 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:198 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:199 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:200 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:201 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:202 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:203 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:204 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:205 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:206 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:207 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:208 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:209 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:210 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:211 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:212 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:213 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:214 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:215 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:216 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:217 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:218 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:219 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:220 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:221 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:222 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:223 */ - -/* local write swap offsets a */ - -/* local write swap offsets b */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:224 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:225 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:226 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:227 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:228 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:229 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:230 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:231 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:232 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:233 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:234 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:235 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:236 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:237 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:238 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:239 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:240 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:241 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:242 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:243 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:244 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:245 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:246 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:247 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:248 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:249 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:250 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:251 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:252 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:253 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:254 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:255 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=1 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -label_Summation_End_OptNLL: -s_cmpk_eq_u32 s[sgprBeta], 0x0 // Beta == 0 -s_cbranch_scc1 label_NoBranch_PAGZHD9H2DI57HHE_0 // Only branch on scc0 -s_getpc_b64 s[80:81] // addr of next instr -s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset -s_add_u32 s80, s80, s82 // add target branch offset -s_addc_u32 s81, s81, 0 // add high and carry -s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd -label_NoBranch_PAGZHD9H2DI57HHE_0: - -s_cmp_eq_u32 s[sgprAlpha], 1.0 // Alpha == 1.0 ? -s_getpc_b64 s[80:81] // addr of next instr -s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset -s_add_u32 s80, s80, s82 // add target branch offset -s_addc_u32 s81, s81, 0 // add high and carry -s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd -label_NoBranch_XQG82FMXJOJL8OIW_0: - -s_and_b32 s80, 255, s[sgprSizeI] // s80 = s[sgprSizeI] % 256 -s_add_u32 s81, -0x1, s[sgprNumWorkGroups0] -s_cmp_ge_u32 s[sgprWorkGroup0], s81 // wg0 >= nwg0-1 ? -s_cselect_b32 s80, s80, 0 // set rMT0 -s_cmpk_gt_u32 s80, 0x0 // rMT0 > 0 -s_cbranch_scc0 label_NoBranch_XQ75AI1RJ5F179IN_0 // Only branch on scc1 -// jump if edges required -s_getpc_b64 s[80:81] // addr of next instr -s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset -s_add_u32 s80, s80, s82 // add target branch offset -s_addc_u32 s81, s81, 0 // add high and carry -s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd -label_NoBranch_XQ75AI1RJ5F179IN_0: -s_and_b32 s80, 255, s[sgprSizeJ] // s80 = s[sgprSizeJ] % 256 -s_add_u32 s81, -0x1, s[sgprNumWorkGroups1] -s_cmp_ge_u32 s[sgprWorkGroup1], s81 // wg1 >= nwg1-1 -s_cselect_b32 s80, s80, 0 // set rMT1 -s_cmpk_gt_u32 s80, 0x0 // rMT1 > 0 -s_cbranch_scc0 label_NoBranch_GFF6GDU2NIUMLQ8E_0 // Only branch on scc1 -// jump if edges required -s_getpc_b64 s[80:81] // addr of next instr -s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset -s_add_u32 s80, s80, s82 // add target branch offset -s_addc_u32 s81, s81, 0 // add high and carry -s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd -label_NoBranch_GFF6GDU2NIUMLQ8E_0: - -s_and_b32 s81, 63, s[sgprSizesSum+0] // s81 = s[sgprSizesSum+0] % 64 -s_cmp_eq_u32 s81, 0x0 // numIterL == 0 -s_cbranch_scc1 label_NoBranch_VEIX0X0UO5PRLIZN_0 // Only branch on scc0 -s_getpc_b64 s[80:81] // addr of next instr -s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset -s_add_u32 s80, s80, s82 // add target branch offset -s_addc_u32 s81, s81, 0 // add high and carry -s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd -label_NoBranch_VEIX0X0UO5PRLIZN_0: - -/* endSummation: add vgpr [0...230) to pool */ -/* load store sgprs */ -.set sgprAddressScaleAlphaVec, 48 -.set sgprAddressBias, 50 -.set sgprBiasType, 52 -.set sgprBiasStride, 53 -.set sgpractivationAlpha, 54 -.set sgpractivationBeta, 55 -.set sgprActivationType, 56 -/* Check if custom structure pointer is null */ -s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? -s_cbranch_scc1 label_LoadExternalEpilogueStruct // branch if ArgType == 2 -s_load_dwordx8 s[48:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x58 -s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x78 -s_branch label_LoadExternalEpilogueStructEnd -label_LoadExternalEpilogueStruct: -s_load_dwordx4 s[48:51], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x90 -s_load_dwordx2 s[52:53], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xa0 -s_load_dwordx2 s[54:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xb8 -s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0xc0 -label_LoadExternalEpilogueStructEnd: -.set sgprSrdScaleAlphaVec, 32 -.set sgprSrdBias, 40 - -/* Mapping of Acc register -> C Vgpr register */ -/* computeStoreVgprs */ -v_lshrrev_b32 v4, 6, v[vgprSerial] // v4 = v[vgprSerial] / 64 -v_lshrrev_b32 v5, 2, v4 // v5 = v4 / 4 -v_mul_lo_u32 v5, 0x10, v5 // wave coordination offset 1 -v_and_b32 v1, 63, v[vgprSerial] // v1 = v[vgprSerial] % 64 -v_lshrrev_b32 v1, 4, v1 // v1 = v1 / 16 -v_lshlrev_b32 v1, 0x2, v1 // thread0 * continuous_output -v_add_lshl_u32 v1, v5, v1, 0 // coordination 1 = vwB *(wave_id1 + tid1) -v_mul_lo_u32 v2, v1, s[sgprStrideC1J] // offset 1 -v_mul_lo_u32 v3, v1, s[sgprStrideD1J] // offset 1 -v_and_b32 v0, 3, v4 // v0 = v4 % 4 -v_mul_lo_u32 v0, 0x10, v0 // wave coordination offset 0 -v_and_b32 v5, 15, v[vgprSerial] // v5 = v[vgprSerial] % 16 -v_add_lshl_u32 v0, v5, v0, 2 // coordination 0 = vwA * (wave_id0 + tid0) -s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_add_u32 v0, s8, v0 // coord 0 = (tid0/MI_m)*4 + waveG0*MIB_m + MT0*SG0 -s_mul_i32 s8, 256, s[sgprWorkGroup1] // wgp1 * MT1 -v_add_u32 v1, s8, v1 // coord 1 = (tid0%MI_m) + waveG1*MIB_n + MT1*SG1 - -/******************************************/ -/* Global Write Elements */ -/******************************************/ -s_waitcnt lgkmcnt(0) // wait for 36 bytes of kern args. -s_mov_b32 s[sgprSrdScaleAlphaVec+0], s[sgprAddressScaleAlphaVec+0] // init SRD base address (lower) -s_mov_b32 s[sgprSrdScaleAlphaVec+1], s[sgprAddressScaleAlphaVec+1] // init SRD base address (upper) + other fields -s_mov_b32 s[sgprSrdScaleAlphaVec+3], Srd127_96 // Set bits 127_96 in post-loop SRD -s_cmp_eq_u64 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], 0 // s[AddressScaleAlphaVec] == 0 ? -s_cbranch_scc0 label_ScaleAlphaVecAddrValid // branch if s[AddressScaleAlphaVec] != 0 -s_mov_b32 s[sgprSrdScaleAlphaVec+2], 0 -s_branch label_ScaleAlphaVecAddrValid_End -label_ScaleAlphaVecAddrValid: -s_mov_b32 s[sgprSrdScaleAlphaVec+2], s[sgprSizeI] -label_ScaleAlphaVecAddrValid_End: - -s_mul_i32 s[sgprSrdScaleAlphaVec+2], 0x4, s[sgprSrdScaleAlphaVec+2] // ScaleAlphaVec scaled by BPE -s_add_u32 s8, s[sgprWorkGroup2], 0x1 -s_mul_i32 s8, s[sgprBiasStride], s8 // stride * (wg+1) -s_cmp_eq_u32 s8, 0x0 // bias stride = 0? -s_cselect_b32 s8, s[sgprSizeI], s8 -s_mov_b32 s[sgprSrdBias+0], s[sgprAddressBias+0] // init SRD base address (lower) -s_mov_b32 s[sgprSrdBias+1], s[sgprAddressBias+1] // init SRD base address (upper) + other fields -s_mov_b32 s[sgprSrdBias+3], Srd127_96 // Set bits 127_96 in post-loop SRD -s_cmp_eq_u64 s[sgprAddressBias:sgprAddressBias+1], 0 // s[AddressBias] == 0 ? -s_cbranch_scc0 label_BiasAddrValid // branch if s[AddressBias] != 0 -s_mov_b32 s[sgprSrdBias+2], 0 -s_branch label_BiasAddrValid_End -label_BiasAddrValid: -s_mov_b32 s[sgprSrdBias+2], s8 -label_BiasAddrValid_End: - -label_Load_Biasf32_0: -s_cmpk_lg_u32 s[sgprBiasType], 0 // BiasType != 0 -s_cbranch_scc1 label_Load_Biasbf16_0 // Branch if true - -/******************************************/ -/* Read Bias to LDS */ -/******************************************/ -s_mul_i32 s[sgprSrdBias+2], 0x4, s[sgprSrdBias+2] // scaled by BPE -s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset -s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG -v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG -v_lshlrev_b32 v8, 0x2, v8 // Global bias address scaled by BPE -buffer_load_dword v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias -v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE -s_waitcnt vmcnt(0) // wait for bias load -s_barrier // Wait for all wavefronts -ds_write_b32 v8, v4 offset:0 // store bias -s_branch label_Load_Bias_End // Branch to load bias end -label_Load_Biasbf16_0: -s_cmpk_lg_u32 s[sgprBiasType], 7 // BiasType != 7 -s_cbranch_scc1 label_Load_Bias_End // Branch if true - -/******************************************/ -/* Read Bias to LDS */ -/******************************************/ -s_mul_i32 s[sgprSrdBias+2], 0x2, s[sgprSrdBias+2] // scaled by BPE -s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset -s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG -v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG -v_lshlrev_b32 v8, 0x1, v8 // Global bias address scaled by BPE -buffer_load_short_d16 v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias -v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE -s_waitcnt vmcnt(0) // wait for bias load -s_barrier // Wait for all wavefronts -v_lshlrev_b32 v4, 16, v4 // cvt bf16 to fp32. -ds_write_b32 v8, v4 offset:0 // store bias -s_branch label_Load_Bias_End // Branch to load bias end -label_Load_Bias_End: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW4 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW4 // Branch if true -label_To_Activation_None_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Abs_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Clippedrelu_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Gelu_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Leakyrelu_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Relu_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Sigmoid_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Tanh_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Geluscaling_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_To_Activation_Silu_VW4: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd -label_ActivationSetPCAddrEnd: -label_GW_B0_E0: - -/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b128 v[20:23], v15 offset:0 // load bias -v_lshlrev_b32 v16, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_lshl_u32 v13, v3, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 -v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+32], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+33], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+34], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+35], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+36], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+37], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+38], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+39], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+40], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+41], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+42], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+43], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+44], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+45], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+46], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+47], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+48], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+49], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+50], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+51], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+52], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+53], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+54], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+55], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+56], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+57], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+58], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+59], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+60], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+61], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+62], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+63], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+64], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+65], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+66], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+67], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+68], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+69], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+70], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+71], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+72], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+73], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+74], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+75], acc47 // copy acc to vreg[47] -v_accvgpr_read_b32 v[vgprValuC+76], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+77], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+78], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+79], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+80], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+81], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+82], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+83], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+84], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+85], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+86], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+87], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+88], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+89], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+90], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+91], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+28], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+29], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+31], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+32], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+33], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+34], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+35], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+36], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+37], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+38], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+39], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+40], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+41], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+42], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+43], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+44], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+45], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+46], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+47], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+48], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+49], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+50], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+51], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+52], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+53], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+54], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+55], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+56], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+57], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+58], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+59], acc95 // copy acc to vreg[95] -v_accvgpr_read_b32 v[vgprValuC+60], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+61], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+62], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+63], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+64], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+65], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+66], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+67], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+68], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+69], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+70], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+71], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+72], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+73], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+74], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+75], acc111 // copy acc to vreg[111] -v_accvgpr_read_b32 v[vgprValuC+76], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+77], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+78], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+79], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+80], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+81], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+82], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+83], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+84], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+85], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+86], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+87], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+88], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+89], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+90], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+91], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+28], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+29], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+31], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+32], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+33], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+34], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+35], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+36], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+37], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+38], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+39], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+40], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+41], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+42], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+43], acc143 // copy acc to vreg[143] -v_accvgpr_read_b32 v[vgprValuC+44], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+45], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+46], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+47], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+48], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+49], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+50], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+51], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+52], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+53], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+54], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+55], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+56], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+57], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+58], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+59], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+60], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+61], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+62], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+63], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+64], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+65], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+66], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+67], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+68], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+69], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+70], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+71], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+72], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+73], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+74], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+75], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+76], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+77], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+78], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+79], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+80], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+81], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+82], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+83], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+84], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+85], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+86], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+87], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+88], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+89], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+90], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+91], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+32], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+33], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+34], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+35], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+36], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+37], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+38], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+39], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+40], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+41], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+42], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+43], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+44], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+45], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+46], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+47], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+48], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+49], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+50], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+51], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+52], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+53], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+54], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+55], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+56], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+57], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+58], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+59], acc223 // copy acc to vreg[223] -v_accvgpr_read_b32 v[vgprValuC+60], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+61], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+62], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+63], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+64], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+65], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+66], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+67], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+68], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+69], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+70], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+71], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+72], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+73], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+74], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+75], acc239 // copy acc to vreg[239] -v_accvgpr_read_b32 v[vgprValuC+76], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+77], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+78], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+79], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+80], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+81], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+82], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+83], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+84], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+85], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+86], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+87], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+88], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+89], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+90], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+91], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End // jump to end -label_GW_End: - -s_endpgm // Kernel End - -label_LoopEndL_odd: -.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 - -/* Before NLL: Check VGPR.checkin for INT8 LW */ - -/******************************************/ -/* Ord. NoGlobalLoadLoop - Begin */ -/******************************************/ -s_waitcnt vmcnt(8) -/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ - -/* iter 0 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:0 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:1 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+0:vgprValuA_X0_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+0:vgprValuA_X2_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc B loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:2 */ -ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:3 */ -s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:4 */ -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:5 */ -ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:6 */ -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:7 */ -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:8 */ -ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:9 */ -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:10 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:11 */ -ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:12 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:13 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:14 */ -ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:15 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:16 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:17 */ -ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:18 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:19 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:20 */ -ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:21 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:22 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:23 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:24 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:25 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:26 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:27 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:28 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:29 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -/* mfmaIndex:30 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:31 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:32 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+4:vgprValuA_X0_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+4:vgprValuA_X2_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:33 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:34 */ -/* schedule remaining localreads for 1LDSB */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:35 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:36 */ -ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:37 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:38 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:39 */ -ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:40 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:41 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:42 */ -ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:43 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:44 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:45 */ -ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:46 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:47 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:48 */ -ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:49 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:50 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:51 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:52 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:53 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:54 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:55 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:56 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:57 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:58 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:59 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:60 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:61 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:62 */ -/* 1 LDS buffer: read-sync-write */ -s_waitcnt lgkmcnt(0) -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:63 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 1 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:64 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:65 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+8:vgprValuA_X0_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+8:vgprValuA_X2_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:66 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:67 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:68 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:69 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:70 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:71 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:72 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:73 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:74 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:75 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:76 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:77 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:78 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:79 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:80 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:81 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:82 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:83 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:84 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:85 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:86 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:87 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:88 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:89 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:90 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:91 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:92 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:93 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:94 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:95 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:96 */ -buffer_load_dwordx4 v[vgprValuA_X0_I0_0+12:vgprValuA_X0_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_1_0 -buffer_load_dwordx4 v[vgprValuA_X2_I0_0+12:vgprValuA_X2_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 -/* global read inc A loopL */ -s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:97 */ -s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:98 */ -s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:99 */ -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:100 */ -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:101 */ -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:102 */ -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:103 */ -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:104 */ -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:105 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:106 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:107 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:108 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:109 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:110 */ -s_waitcnt vmcnt(12) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:111 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:112 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:113 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:114 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:115 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:116 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:117 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:118 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:119 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:120 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:121 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:122 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:123 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:124 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:125 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:126 */ -s_waitcnt vmcnt(11) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:127 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:128 */ -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:129 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:130 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:131 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:132 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:133 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:134 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:135 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:136 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:137 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:138 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:139 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:140 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:141 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:142 */ -s_waitcnt vmcnt(10) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:143 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:144 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:145 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:146 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:147 */ -/* sched write - iter 2 writesPerItem=1 */ -/* sched write - iter 2 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:148 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:149 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:150 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:151 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:152 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:153 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:154 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:155 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:156 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:157 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:158 */ -s_waitcnt vmcnt(9) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:159 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:160 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:161 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:162 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:163 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:164 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:165 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:166 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:167 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:168 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:169 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:170 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:171 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:172 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:173 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:174 */ -s_waitcnt vmcnt(8) // wait for global read before writing to local -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:175 */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:176 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:177 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:178 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:179 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:180 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:181 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:182 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:183 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:184 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:185 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:186 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:187 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:188 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:189 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:190 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:191 */ - -/* local read swap offsets a */ - -/* local read swap offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ - -/* iter 3 (swap and reset local write pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:192 */ -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:193 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:194 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:195 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:196 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:197 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:198 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:199 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:200 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:201 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:202 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:203 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:204 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:205 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:206 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:207 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:208 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:209 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:210 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:211 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:212 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:213 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:214 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:215 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:216 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:217 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:218 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:219 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:220 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:221 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:222 */ - -/* local write swap offsets a */ - -/* local write swap offsets b */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 -s_barrier -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:223 */ -ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:224 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:225 */ -ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:226 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:227 */ -ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:228 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:229 */ -ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:230 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:231 */ -ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:232 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:233 */ -ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:234 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:235 */ -ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:236 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:237 */ -ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:238 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:239 */ -ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:240 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:241 */ -ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:242 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:243 */ -ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:244 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:245 */ -ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:246 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:247 */ -ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:248 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:249 */ -ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:250 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:251 */ -ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:252 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:253 */ -ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:254 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:255 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=1 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 - -s_branch label_LoopEndL_odd_NoLoadLoop - -label_LoopEndL_odd_NoLoadLoop: - -/******************************************/ -/* Ord. NoLoadLoop - Begin */ -/******************************************/ -s_waitcnt vmcnt(0) - -/* iter 0 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:0 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:1 */ -ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:2 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:3 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:4 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:5 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:6 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:7 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:8 */ -ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:9 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:10 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:11 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:12 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:13 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:14 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:15 */ -ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:16 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:17 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:18 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:19 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:20 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:21 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:22 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:23 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:24 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:25 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:26 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:27 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:28 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:29 */ -/* localReadsVacancy: latencyLeft 2 */ -ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:30 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:31 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:32 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:33 */ -/* localReadsVacancy: latencyLeft 2 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:34 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:35 */ -/* sched write - iter 0 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:36 */ -ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:37 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:38 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:39 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:40 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:41 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:42 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:43 */ -ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:44 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:45 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:46 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:47 */ -/* sched write - iter 0 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:48 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:49 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:50 */ -ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:51 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:52 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:53 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:54 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:55 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:56 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:57 */ -ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:58 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:59 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:60 */ -/* sched write - iter 0 writesPerItem=1 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:61 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:62 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:63 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 1 */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:64 */ -ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:65 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:66 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:67 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:68 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:69 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:70 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:71 */ -ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:72 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:73 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:74 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:75 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:76 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:77 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:78 */ -ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:79 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:80 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:81 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:82 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:83 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:84 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:85 */ -ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:86 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:87 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:88 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:89 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:90 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:91 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:92 */ -ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:93 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:94 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:95 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:96 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:97 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:98 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:99 */ -ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:100 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:101 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:102 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:103 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:104 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:105 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:106 */ -ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:107 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:108 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:109 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:110 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:111 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:112 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:113 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:114 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:115 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:116 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:117 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:118 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:119 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:120 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:121 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:122 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:123 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:124 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:125 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:126 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:127 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:128 */ -s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:129 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:130 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:131 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:132 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:133 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:134 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:135 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:136 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:137 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:138 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:139 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:140 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:141 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:142 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:143 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:144 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:145 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:146 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:147 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:148 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:149 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:150 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:151 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:152 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:153 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:154 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:155 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:156 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:157 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:158 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:159 */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:160 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:161 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:162 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:163 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:164 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:165 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:166 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:167 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:168 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:169 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:170 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:171 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:172 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:173 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:174 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:175 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:176 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:177 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:178 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:179 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:180 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:181 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:182 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:183 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:184 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:185 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:186 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:187 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:188 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:189 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:190 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:191 */ - -/* local read swap offsets a */ - -/* local read swap offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=0 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ - -/* iter 3 (swap and reset local write pointers iteration) */ -/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ -/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ -/* mfmaIndex:192 */ -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] -/* mfmaIndex:193 */ -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] -/* mfmaIndex:194 */ -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] -/* mfmaIndex:195 */ -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] -/* mfmaIndex:196 */ -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] -/* mfmaIndex:197 */ -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] -/* mfmaIndex:198 */ -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] -/* mfmaIndex:199 */ -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] -/* mfmaIndex:200 */ -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] -/* mfmaIndex:201 */ -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] -/* mfmaIndex:202 */ -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] -/* mfmaIndex:203 */ -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] -/* mfmaIndex:204 */ -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] -/* mfmaIndex:205 */ -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] -/* mfmaIndex:206 */ -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] -/* mfmaIndex:207 */ -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] -/* mfmaIndex:208 */ -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] -/* mfmaIndex:209 */ -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] -/* mfmaIndex:210 */ -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] -/* mfmaIndex:211 */ -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] -/* mfmaIndex:212 */ -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] -/* mfmaIndex:213 */ -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] -/* mfmaIndex:214 */ -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] -/* mfmaIndex:215 */ -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] -/* mfmaIndex:216 */ -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] -/* mfmaIndex:217 */ -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] -/* mfmaIndex:218 */ -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] -/* mfmaIndex:219 */ -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] -/* mfmaIndex:220 */ -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] -/* mfmaIndex:221 */ -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] -/* mfmaIndex:222 */ -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] -/* mfmaIndex:223 */ - -/* local write swap offsets a */ - -/* local write swap offsets b */ -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] -/* mfmaIndex:224 */ -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] -/* mfmaIndex:225 */ -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] -/* mfmaIndex:226 */ -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] -/* mfmaIndex:227 */ -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] -/* mfmaIndex:228 */ -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] -/* mfmaIndex:229 */ -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] -/* mfmaIndex:230 */ -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] -/* mfmaIndex:231 */ -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] -/* mfmaIndex:232 */ -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] -/* mfmaIndex:233 */ -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] -/* mfmaIndex:234 */ -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] -/* mfmaIndex:235 */ -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] -/* mfmaIndex:236 */ -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] -/* mfmaIndex:237 */ -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] -/* mfmaIndex:238 */ -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] -/* mfmaIndex:239 */ -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] -/* mfmaIndex:240 */ -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] -/* mfmaIndex:241 */ -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] -/* mfmaIndex:242 */ -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] -/* mfmaIndex:243 */ -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] -/* mfmaIndex:244 */ -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] -/* mfmaIndex:245 */ -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] -/* mfmaIndex:246 */ -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] -/* mfmaIndex:247 */ -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] -/* mfmaIndex:248 */ -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] -/* mfmaIndex:249 */ -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] -/* mfmaIndex:250 */ -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] -/* mfmaIndex:251 */ -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] -/* mfmaIndex:252 */ -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] -/* mfmaIndex:253 */ -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] -/* mfmaIndex:254 */ -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] -/* mfmaIndex:255 */ -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] -/* numPrefetchIter=1 */ -/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ -/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ - -s_branch label_Summation_End_OptNLL - -label_PrefetchGlobalLastIterEnd: -.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 -.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 - -/******************************************/ -/* Tail Loop */ -/******************************************/ - -/* Tail: add ValuA/B vgpr buffer [0...160) to pool */ - -/* local write reset offsets a */ - -/* local write reset offsets b */ - -// numIterL = LOCAL_SPLITU * min(sizeL % LOCAL_DEPTHU, DEPTHU / LOCAL_SPLITU) -s_and_b32 s[sgprLoopCounterL], 63, s[sgprSizesSum+0] // s[sgprLoopCounterL] = s[sgprSizesSum+0] % 64 -s_cmp_lg_u32 s[sgprGSUSumIdx], s[sgprGSUSumIdx+1] // gsuSumIdx == numIterPerWgRemainder -s_cmov_b32 s[sgprLoopCounterL], 0x0 // numIter=0 if gsuSimIdx!=remainder -s_cmp_eq_u32 s[sgprLoopCounterL], 0x0 // numIterL == 0 -s_mov_b32 s[sgprOrigLoopCounter], 0 // repurpose to count each localRead increment -s_cbranch_scc1 label_SkipTailLoopL // skip to end of tail loop b/c numIter==0 - -/* remove stagger offsets for tail loop */ -s_sub_i32 s80, 3, s[sgprStaggerUIter] -s_mul_hi_i32 s81, s80, s[sgprGlobalReadIncsA+0] // start offset S in bytes -s_mul_i32 s80, s80, s[sgprGlobalReadIncsA+0] // start offset S in bytes -s_sub_u32 s80, s80, s[sgprWrapUA] // S - WrapU -s_subb_u32 s81, s81, s[sgprWrapUA+1] // S - WrapU -s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) -s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) -s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) -s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) -s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 -s_sub_i32 s80, 3, s[sgprStaggerUIter] -s_mul_hi_i32 s81, s80, s[sgprGlobalReadIncsB+0] // start offset S in bytes -s_mul_i32 s80, s80, s[sgprGlobalReadIncsB+0] // start offset S in bytes -s_sub_u32 s80, s80, s[sgprWrapUB] // S - WrapU -s_subb_u32 s81, s81, s[sgprWrapUB+1] // S - WrapU -s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) -s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) -s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) -s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) -s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? -s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 - -/* Recalc global read offsets */ -v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v0 15, v1 // 1. M offset: mIdx = wtid % MI_M(16) -v_mul_lo_u32 v0, s[sgprStrideA0I], v0 // 1. M offset: mOffset = mIdx * mStride(k) -v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) -v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v1, 0x2, v1 // 5. K offset: lrKOffset = kIdx * mStride(4) -v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset -v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in M dimen: wtid = tid / dividedForWaveId(64) -v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid % num1DWaves(4) -v_mul_lo_u32 v1, s[sgprStrideA0I], v1 // 7. wave offset in M dimen: wOffset = wtid0 * s[sgprStrideA0I](8192) -v_lshlrev_b32 v1, 0x6, v1 // 7. wave offset in M dimen: wOffset = wOffset * 16 * vw(4) -v_add_u32 v[vgprGlobalReadOffsetA], v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset -v_add_u32 v[vgprGlobalReadOffsetA] 0x8 v[vgprGlobalReadOffsetA] // add prepad for pointer shift -v_lshlrev_b32 v[vgprGlobalReadOffsetA] 0x1 v[vgprGlobalReadOffsetA] // offset *= bytes/element - -s_mul_i32 s[sgprScalarGlobalReadOffsetA+0], s[sgprStrideA0I], 1 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+0], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetA+1], s[sgprStrideA0I], 2 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+1], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetA+2], s[sgprStrideA0I], 3 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+2], 0x1 // scalar offset *= bytes/element -s_mul_i32 s[sgprScalarGlobalReadOffsetA+3], 1, 16 // compute offset diff (scaled tileDim) -s_lshl_b32 s[sgprScalarGlobalReadOffsetA+3], s[sgprScalarGlobalReadOffsetA+3], 0x1 // scalar offset *= bytes/element -s_add_u32 s[sgprScalarGlobalReadOffsetA+4], s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+3] -s_add_u32 s[sgprScalarGlobalReadOffsetA+5], s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+3] -s_add_u32 s[sgprScalarGlobalReadOffsetA+6], s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+3] - -/* Update M0 for DTLDS */ - -/* g2l=0, load component 0 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+0], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // load one buffer value -/* g2l=0, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+0], v[vgprValuA_X0_I0+0], v0 // HasEccHalf: pack -/* g2l=0, load component 2 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+1], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:4 // load one buffer value -/* g2l=0, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+1], v[vgprValuA_X0_I0+1], v0 // HasEccHalf: pack -/* g2l=0, load component 0 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+8], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // load one buffer value -/* g2l=0, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+8], v[vgprValuA_X0_I0+8], v0 // HasEccHalf: pack -/* g2l=0, load component 2 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+9], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:4 // load one buffer value -/* g2l=0, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+9], v[vgprValuA_X0_I0+9], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+3], s[sgprScalarGlobalReadOffsetA+3], 64 -/* g2l=0, load component 0 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+0], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:64 // load one buffer value -/* g2l=0, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:66 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+0], v[vgprValuA_X2_I0+0], v0 // HasEccHalf: pack -/* g2l=0, load component 2 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+1], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:68 // load one buffer value -/* g2l=0, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:70 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+1], v[vgprValuA_X2_I0+1], v0 // HasEccHalf: pack -/* g2l=0, load component 0 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+8], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // load one buffer value -/* g2l=0, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+8], v[vgprValuA_X2_I0+8], v0 // HasEccHalf: pack -/* g2l=0, load component 2 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+9], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:4 // load one buffer value -/* g2l=0, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+9], v[vgprValuA_X2_I0+9], v0 // HasEccHalf: pack -/* g2l=0, load component 4 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+2], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // load one buffer value -/* g2l=0, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+2], v[vgprValuA_X0_I0+2], v0 // HasEccHalf: pack -/* g2l=0, load component 6 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:4 // load one buffer value -/* g2l=0, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+3], v[vgprValuA_X0_I0+3], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+0], 64 -/* g2l=0, load component 4 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+10], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // load one buffer value -/* g2l=0, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+10], v[vgprValuA_X0_I0+10], v0 // HasEccHalf: pack -/* g2l=0, load component 6 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+11], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:4 // load one buffer value -/* g2l=0, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+11], v[vgprValuA_X0_I0+11], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+4], s[sgprScalarGlobalReadOffsetA+4], 64 -/* g2l=0, load component 4 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+2], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // load one buffer value -/* g2l=0, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+2], v[vgprValuA_X2_I0+2], v0 // HasEccHalf: pack -/* g2l=0, load component 6 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:4 // load one buffer value -/* g2l=0, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+3], v[vgprValuA_X2_I0+3], v0 // HasEccHalf: pack -/* g2l=0, load component 4 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+10], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // load one buffer value -/* g2l=0, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+10], v[vgprValuA_X2_I0+10], v0 // HasEccHalf: pack -/* g2l=0, load component 6 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+11], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:4 // load one buffer value -/* g2l=0, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+11], v[vgprValuA_X2_I0+11], v0 // HasEccHalf: pack -/* g2l=0, load component 8 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+4], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // load one buffer value -/* g2l=0, load component 9 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+4], v[vgprValuA_X0_I0+4], v0 // HasEccHalf: pack -/* g2l=0, load component 10 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+5], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:4 // load one buffer value -/* g2l=0, load component 11 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+5], v[vgprValuA_X0_I0+5], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+1], 64 -/* g2l=0, load component 8 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+12], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // load one buffer value -/* g2l=0, load component 9 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+12], v[vgprValuA_X0_I0+12], v0 // HasEccHalf: pack -/* g2l=0, load component 10 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+13], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:4 // load one buffer value -/* g2l=0, load component 11 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+13], v[vgprValuA_X0_I0+13], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+5], s[sgprScalarGlobalReadOffsetA+5], 64 -/* g2l=0, load component 8 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+4], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // load one buffer value -/* g2l=0, load component 9 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+4], v[vgprValuA_X2_I0+4], v0 // HasEccHalf: pack -/* g2l=0, load component 10 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+5], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:4 // load one buffer value -/* g2l=0, load component 11 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+5], v[vgprValuA_X2_I0+5], v0 // HasEccHalf: pack -/* g2l=0, load component 8 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+12], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // load one buffer value -/* g2l=0, load component 9 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+12], v[vgprValuA_X2_I0+12], v0 // HasEccHalf: pack -/* g2l=0, load component 10 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+13], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:4 // load one buffer value -/* g2l=0, load component 11 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+13], v[vgprValuA_X2_I0+13], v0 // HasEccHalf: pack -/* g2l=0, load component 12 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+6], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // load one buffer value -/* g2l=0, load component 13 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+6], v[vgprValuA_X0_I0+6], v0 // HasEccHalf: pack -/* g2l=0, load component 14 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+7], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:4 // load one buffer value -/* g2l=0, load component 15 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+7], v[vgprValuA_X0_I0+7], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+2], 64 -/* g2l=0, load component 12 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+14], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // load one buffer value -/* g2l=0, load component 13 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+14], v[vgprValuA_X0_I0+14], v0 // HasEccHalf: pack -/* g2l=0, load component 14 */ -buffer_load_short_d16 v[vgprValuA_X0_I0+15], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:4 // load one buffer value -/* g2l=0, load component 15 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X0_I0+15], v[vgprValuA_X0_I0+15], v0 // HasEccHalf: pack -s_add_u32 s[sgprScalarGlobalReadOffsetA+6], s[sgprScalarGlobalReadOffsetA+6], 64 -/* g2l=0, load component 12 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+6], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // load one buffer value -/* g2l=0, load component 13 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+6], v[vgprValuA_X2_I0+6], v0 // HasEccHalf: pack -/* g2l=0, load component 14 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+7], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:4 // load one buffer value -/* g2l=0, load component 15 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+7], v[vgprValuA_X2_I0+7], v0 // HasEccHalf: pack -/* g2l=0, load component 12 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+14], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // load one buffer value -/* g2l=0, load component 13 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+14], v[vgprValuA_X2_I0+14], v0 // HasEccHalf: pack -/* g2l=0, load component 14 */ -buffer_load_short_d16 v[vgprValuA_X2_I0+15], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:4 // load one buffer value -/* g2l=0, load component 15 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprValuA_X2_I0+15], v[vgprValuA_X2_I0+15], v0 // HasEccHalf: pack - -/* Update M0 for DTLDS */ - -/* global read B */ -/* g2l=0, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+0+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // load one buffer value -/* g2l=0, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+0+0], v[vgprG2LB+0+0], v0 // HasEccHalf: pack -/* g2l=0, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+0+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:4 // load one buffer value -/* g2l=0, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+0+1], v[vgprG2LB+0+1], v0 // HasEccHalf: pack -/* g2l=0, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+0+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:8 // load one buffer value -/* g2l=0, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+0+2], v[vgprG2LB+0+2], v0 // HasEccHalf: pack -/* g2l=0, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:12 // load one buffer value -/* g2l=0, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+0+3], v[vgprG2LB+0+3], v0 // HasEccHalf: pack -/* g2l=4, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+4+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // load one buffer value -/* g2l=4, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+4+0], v[vgprG2LB+4+0], v0 // HasEccHalf: pack -/* g2l=4, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+4+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:4 // load one buffer value -/* g2l=4, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+4+1], v[vgprG2LB+4+1], v0 // HasEccHalf: pack -/* g2l=4, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+4+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:8 // load one buffer value -/* g2l=4, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+4+2], v[vgprG2LB+4+2], v0 // HasEccHalf: pack -/* g2l=4, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:12 // load one buffer value -/* g2l=4, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+4+3], v[vgprG2LB+4+3], v0 // HasEccHalf: pack -/* g2l=8, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+8+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // load one buffer value -/* g2l=8, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+8+0], v[vgprG2LB+8+0], v0 // HasEccHalf: pack -/* g2l=8, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+8+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:4 // load one buffer value -/* g2l=8, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+8+1], v[vgprG2LB+8+1], v0 // HasEccHalf: pack -/* g2l=8, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+8+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:8 // load one buffer value -/* g2l=8, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+8+2], v[vgprG2LB+8+2], v0 // HasEccHalf: pack -/* g2l=8, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:12 // load one buffer value -/* g2l=8, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+8+3], v[vgprG2LB+8+3], v0 // HasEccHalf: pack -/* g2l=12, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+12+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // load one buffer value -/* g2l=12, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+12+0], v[vgprG2LB+12+0], v0 // HasEccHalf: pack -/* g2l=12, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+12+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:4 // load one buffer value -/* g2l=12, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+12+1], v[vgprG2LB+12+1], v0 // HasEccHalf: pack -/* g2l=12, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+12+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:8 // load one buffer value -/* g2l=12, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+12+2], v[vgprG2LB+12+2], v0 // HasEccHalf: pack -/* g2l=12, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:12 // load one buffer value -/* g2l=12, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+12+3], v[vgprG2LB+12+3], v0 // HasEccHalf: pack -/* g2l=16, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+16+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // load one buffer value -/* g2l=16, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+16+0], v[vgprG2LB+16+0], v0 // HasEccHalf: pack -/* g2l=16, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+16+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:4 // load one buffer value -/* g2l=16, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+16+1], v[vgprG2LB+16+1], v0 // HasEccHalf: pack -/* g2l=16, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+16+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:8 // load one buffer value -/* g2l=16, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+16+2], v[vgprG2LB+16+2], v0 // HasEccHalf: pack -/* g2l=16, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:12 // load one buffer value -/* g2l=16, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+16+3], v[vgprG2LB+16+3], v0 // HasEccHalf: pack -/* g2l=20, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+20+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // load one buffer value -/* g2l=20, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+20+0], v[vgprG2LB+20+0], v0 // HasEccHalf: pack -/* g2l=20, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+20+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:4 // load one buffer value -/* g2l=20, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+20+1], v[vgprG2LB+20+1], v0 // HasEccHalf: pack -/* g2l=20, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+20+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:8 // load one buffer value -/* g2l=20, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+20+2], v[vgprG2LB+20+2], v0 // HasEccHalf: pack -/* g2l=20, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:12 // load one buffer value -/* g2l=20, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+20+3], v[vgprG2LB+20+3], v0 // HasEccHalf: pack -/* g2l=24, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+24+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // load one buffer value -/* g2l=24, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+24+0], v[vgprG2LB+24+0], v0 // HasEccHalf: pack -/* g2l=24, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+24+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:4 // load one buffer value -/* g2l=24, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+24+1], v[vgprG2LB+24+1], v0 // HasEccHalf: pack -/* g2l=24, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+24+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:8 // load one buffer value -/* g2l=24, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+24+2], v[vgprG2LB+24+2], v0 // HasEccHalf: pack -/* g2l=24, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:12 // load one buffer value -/* g2l=24, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+24+3], v[vgprG2LB+24+3], v0 // HasEccHalf: pack -/* g2l=28, load component 0 */ -buffer_load_short_d16 v[vgprG2LB+28+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // load one buffer value -/* g2l=28, load component 1 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:2 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+28+0], v[vgprG2LB+28+0], v0 // HasEccHalf: pack -/* g2l=28, load component 2 */ -buffer_load_short_d16 v[vgprG2LB+28+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:4 // load one buffer value -/* g2l=28, load component 3 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:6 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+28+1], v[vgprG2LB+28+1], v0 // HasEccHalf: pack -/* g2l=28, load component 4 */ -buffer_load_short_d16 v[vgprG2LB+28+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:8 // load one buffer value -/* g2l=28, load component 5 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:10 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+28+2], v[vgprG2LB+28+2], v0 // HasEccHalf: pack -/* g2l=28, load component 6 */ -buffer_load_short_d16 v[vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:12 // load one buffer value -/* g2l=28, load component 7 */ -buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:14 // load one buffer value -s_waitcnt vmcnt(0) -v_or_b32 v[vgprG2LB+28+3], v[vgprG2LB+28+3], v0 // HasEccHalf: pack -s_waitcnt vmcnt(0) // 2wait for global read -// Skip force waitcnt0 -s_barrier - -/* local write a */ - -/* local write b */ -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 -ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 - -/* Recalc local read offsets */ -/* lr0I */ -v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v0, 15, v1 // 1. N offset: nIdx = wtid % MI_N(16) -v_lshlrev_b32 v0, 0x6, v0 // 1. N offset: nOffset = nIdx * nStride(64) -/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ -v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) -v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v1, 0x2, v1 // 5. K offset: lrKOffset = kIdx * mStride(4) -v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset -v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in N dimen: wtid = tid / dividedForWaveId(64) -v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) -v_lshlrev_b32 v1, 0xc, v1 // 7. wave offset in M dimen: wOffset = wtid0 * W0Stride(4096) -v_add_u32 v0, v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset -/* lr1J */ -v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) -v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) -v_lshlrev_b32 v1, 0x6, v1 // 1. N offset: nOffset = nIdx * nStride(64) -/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ - // 4. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) -v_and_b32 v2, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) -v_lshrrev_b32 v2, 4, v2 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) -v_lshlrev_b32 v2, 0x2, v2 // 5. K offset: lrKOffset = kIdx * mStride(4) -v_add_u32 v1, v2, v1 // 6. offset in wave: lrOffset = bnOffset + lrKOffset -v_lshrrev_b32 v2, 6, v[vgprSerial] // v2 = v[vgprSerial] / 64 -v_lshrrev_b32 v2, 2, v2 // LSU offset: Get LSU wave_id -s_mov_b32 s8, 64 // LSU offset: stride = lsuStride(64) when umlds==True -v_mul_lo_u32 v2, s8, v2 // LSU offset: lsuoffset = wave_id*lsuStride*(MT0+PAD) -v_add_lshl_u32 v[vgprLocalReadAddrA], v2, v0, 0x1 // Final Offset: offset = (lro0+lsuoffset)*bpeDS -v_lshrrev_b32 v3, 9, v[vgprLocalReadAddrA] // Final Offset: padding 32 per block 512 -v_lshlrev_b32 v3, 0x5, v3 // Final Offset: padding 32 per block 512 -v_add_u32 v[vgprLocalReadAddrA], v3, v[vgprLocalReadAddrA] // Final Offset: add padding 32 per block 512 -/* N/A */ -v_lshrrev_b32 v0, 6, v[vgprSerial] // v0 = v[vgprSerial] / 64 -v_lshrrev_b32 v0, 2, v0 // LSU offset: Get LSU wave_id -s_mov_b32 s8, 64 // LSU offset: stride = lsuStride(64) when umlds==True -v_mul_lo_u32 v0, s8, v0 // LSU offset: lsuoffset = wave_id*lsuStride*(MT1+PAD) -v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v1, 0x1 // Final Offset: offset = (lro1+lsuoffset)*bpeDS -v_lshrrev_b32 v2, 7, v[vgprLocalReadAddrB] // Final Offset: padding 32 per block 128 -v_lshlrev_b32 v2, 0x5, v2 // Final Offset: padding 32 per block 128 -v_add_u32 v[vgprLocalReadAddrB], v2, v[vgprLocalReadAddrB] // Final Offset: add padding 32 per block 128 -s_waitcnt lgkmcnt(0) // 5wait for local write -// Skip force waitcnt0 -s_barrier - -/* local read reset offsets a */ - -/* local read reset offsets b */ - -/* local read init pointers a */ - -/* localReadInitPointers */ - -/* local read init pointers b */ - -/* localReadInitPointers */ - -/* tail loop: macs */ -label_TailLoopBeginL: - -/* Tail: remove ValuA/B vgpr buffer [0...160) from pool */ - -/* Tail: add address/G2L vgpr [160...230) to pool */ - -/* local read a */ - -/* local read b */ -ds_read_b64 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+1], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+2:vgprValuB_X0_I0+2+1], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+1], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+6:vgprValuB_X0_I0+6+1], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+1], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+10:vgprValuB_X0_I0+10+1], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+1], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+14:vgprValuB_X0_I0+14+1], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+1], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+18:vgprValuB_X0_I0+18+1], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+1], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+22:vgprValuB_X0_I0+22+1], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+1], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+26:vgprValuB_X0_I0+26+1], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+1], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 -ds_read_b64 v[vgprValuB_X0_I0+30:vgprValuB_X0_I0+30+1], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 - -/* local read inc a */ -s_mov_b32 s8, 0x20 // inc - -/* local read inc b */ -s_mov_b32 s8, 0x20 // inc -v_add_co_u32 v[vgprLocalReadAddrB], vcc, s8, v[vgprLocalReadAddrB] // lrB += 32 (bpeDS) -s_waitcnt lgkmcnt(0) // 4wait for local read -v_and_b32 v160, 63, v[vgprSerial] // v160 = v[vgprSerial] % 64 -v_lshrrev_b32 v160, 4, v160 // v160 = v160 / 16 -v_lshlrev_b32 v160, 0x2, v160 // v160 = v160 * 4 -v_cmp_ge_i32 s[80:81], v160, s[sgprLoopCounterL] // check K index >= Size L -v_cndmask_b32 v[vgprValuA_X0_I0+0+0], v[vgprValuA_X0_I0+0+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+2+0], v[vgprValuA_X0_I0+2+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+4+0], v[vgprValuA_X0_I0+4+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+6+0], v[vgprValuA_X0_I0+6+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+0+1], v[vgprValuA_X0_I0+0+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+2+1], v[vgprValuA_X0_I0+2+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+4+1], v[vgprValuA_X0_I0+4+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuA_X0_I0+6+1], v[vgprValuA_X0_I0+6+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+0+0], v[vgprValuB_X0_I0+0+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+2+0], v[vgprValuB_X0_I0+2+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+4+0], v[vgprValuB_X0_I0+4+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+6+0], v[vgprValuB_X0_I0+6+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+8+0], v[vgprValuB_X0_I0+8+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+10+0], v[vgprValuB_X0_I0+10+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+12+0], v[vgprValuB_X0_I0+12+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+14+0], v[vgprValuB_X0_I0+14+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+16+0], v[vgprValuB_X0_I0+16+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+18+0], v[vgprValuB_X0_I0+18+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+20+0], v[vgprValuB_X0_I0+20+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+22+0], v[vgprValuB_X0_I0+22+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+24+0], v[vgprValuB_X0_I0+24+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+26+0], v[vgprValuB_X0_I0+26+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+28+0], v[vgprValuB_X0_I0+28+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+30+0], v[vgprValuB_X0_I0+30+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+0+1], v[vgprValuB_X0_I0+0+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+2+1], v[vgprValuB_X0_I0+2+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+4+1], v[vgprValuB_X0_I0+4+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+6+1], v[vgprValuB_X0_I0+6+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+8+1], v[vgprValuB_X0_I0+8+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+10+1], v[vgprValuB_X0_I0+10+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+12+1], v[vgprValuB_X0_I0+12+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+14+1], v[vgprValuB_X0_I0+14+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+16+1], v[vgprValuB_X0_I0+16+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+18+1], v[vgprValuB_X0_I0+18+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+20+1], v[vgprValuB_X0_I0+20+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+22+1], v[vgprValuB_X0_I0+22+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+24+1], v[vgprValuB_X0_I0+24+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+26+1], v[vgprValuB_X0_I0+26+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+28+1], v[vgprValuB_X0_I0+28+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_cndmask_b32 v[vgprValuB_X0_I0+30+1], v[vgprValuB_X0_I0+30+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL -v_sub_u32 v160, s[sgprLoopCounterL], v160 // get distance between size and k index -v_cmp_lt_i32 s[80:81], v160, 4 // set partial 0 if distance less than input per thread -s_and_b32 s82, s[sgprLoopCounterL], 3 // get inputs for edge thread -s_sub_u32 s82, 4, s82 // use shift to fill 0 for outside element -s_lshl_b32 s82, s82, 4 // use shift to fill 0 for outside element -v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1] -v_cndmask_b32 v[vgprValuA_X0_I0+0+0+0+0], v[vgprValuA_X0_I0+0+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuA_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1] -v_cndmask_b32 v[vgprValuA_X0_I0+2+0+0+0], v[vgprValuA_X0_I0+2+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuA_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+2+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1] -v_cndmask_b32 v[vgprValuA_X0_I0+4+0+0+0], v[vgprValuA_X0_I0+4+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuA_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1] -v_cndmask_b32 v[vgprValuA_X0_I0+6+0+0+0], v[vgprValuA_X0_I0+6+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuA_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+6+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+0+0+0+0], v[vgprValuB_X0_I0+0+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+0+0+0+1], v[vgprValuB_X0_I0+0+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+2+0+0+0], v[vgprValuB_X0_I0+2+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+2+0+0+1], v[vgprValuB_X0_I0+2+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+4+0+0+0], v[vgprValuB_X0_I0+4+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+4+0+0+1], v[vgprValuB_X0_I0+4+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+6+0+0+0], v[vgprValuB_X0_I0+6+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+6+0+0+1], v[vgprValuB_X0_I0+6+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+8+0+0+0], v[vgprValuB_X0_I0+8+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+8+0+0+1], v[vgprValuB_X0_I0+8+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+10+0+0+0], v[vgprValuB_X0_I0+10+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+10+0+0+1], v[vgprValuB_X0_I0+10+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+12+0+0+0], v[vgprValuB_X0_I0+12+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+12+0+0+1], v[vgprValuB_X0_I0+12+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+14+0+0+0], v[vgprValuB_X0_I0+14+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+14+0+0+1], v[vgprValuB_X0_I0+14+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+16+0+0+0], v[vgprValuB_X0_I0+16+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+16+0+0+1], v[vgprValuB_X0_I0+16+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+18+0+0+0], v[vgprValuB_X0_I0+18+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+18+0+0+1], v[vgprValuB_X0_I0+18+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+20+0+0+0], v[vgprValuB_X0_I0+20+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+20+0+0+1], v[vgprValuB_X0_I0+20+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+22+0+0+0], v[vgprValuB_X0_I0+22+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+22+0+0+1], v[vgprValuB_X0_I0+22+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+24+0+0+0], v[vgprValuB_X0_I0+24+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+24+0+0+1], v[vgprValuB_X0_I0+24+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+26+0+0+0], v[vgprValuB_X0_I0+26+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+26+0+0+1], v[vgprValuB_X0_I0+26+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+28+0+0+0], v[vgprValuB_X0_I0+28+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+28+0+0+1], v[vgprValuB_X0_I0+28+0+0+1], v163, s[80:81] -v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1] -v_cndmask_b32 v[vgprValuB_X0_I0+30+0+0+0], v[vgprValuB_X0_I0+30+0+0+0], v162, s[80:81] -v_cndmask_b32 v[vgprValuB_X0_I0+30+0+0+1], v[vgprValuB_X0_I0+30+0+0+1], v163, s[80:81] -s_nop 1 -v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] -v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[4:7] // left value = acc[4+0:7+0] -v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[8:11] // left value = acc[8+0:11+0] -v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[12:15] // left value = acc[12+0:15+0] -v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] -v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[20:23] // left value = acc[20+0:23+0] -v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[24:27] // left value = acc[24+0:27+0] -v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[28:31] // left value = acc[28+0:31+0] -v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] -v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[36:39] // left value = acc[36+0:39+0] -v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[40:43] // left value = acc[40+0:43+0] -v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[44:47] // left value = acc[44+0:47+0] -v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] -v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[52:55] // left value = acc[52+0:55+0] -v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[56:59] // left value = acc[56+0:59+0] -v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[60:63] // left value = acc[60+0:63+0] -v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] -v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[68:71] // left value = acc[68+0:71+0] -v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[72:75] // left value = acc[72+0:75+0] -v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[76:79] // left value = acc[76+0:79+0] -v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] -v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[84:87] // left value = acc[84+0:87+0] -v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[88:91] // left value = acc[88+0:91+0] -v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[92:95] // left value = acc[92+0:95+0] -v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] -v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[100:103] // left value = acc[100+0:103+0] -v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[104:107] // left value = acc[104+0:107+0] -v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[108:111] // left value = acc[108+0:111+0] -v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] -v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[116:119] // left value = acc[116+0:119+0] -v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[120:123] // left value = acc[120+0:123+0] -v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[124:127] // left value = acc[124+0:127+0] -v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] -v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[132:135] // left value = acc[132+0:135+0] -v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[136:139] // left value = acc[136+0:139+0] -v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[140:143] // left value = acc[140+0:143+0] -v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] -v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[148:151] // left value = acc[148+0:151+0] -v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[152:155] // left value = acc[152+0:155+0] -v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[156:159] // left value = acc[156+0:159+0] -v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] -v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[164:167] // left value = acc[164+0:167+0] -v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[168:171] // left value = acc[168+0:171+0] -v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[172:175] // left value = acc[172+0:175+0] -v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] -v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[180:183] // left value = acc[180+0:183+0] -v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[184:187] // left value = acc[184+0:187+0] -v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[188:191] // left value = acc[188+0:191+0] -v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] -v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[196:199] // left value = acc[196+0:199+0] -v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[200:203] // left value = acc[200+0:203+0] -v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[204:207] // left value = acc[204+0:207+0] -v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] -v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[212:215] // left value = acc[212+0:215+0] -v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[216:219] // left value = acc[216+0:219+0] -v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[220:223] // left value = acc[220+0:223+0] -v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] -v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[228:231] // left value = acc[228+0:231+0] -v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[232:235] // left value = acc[232+0:235+0] -v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[236:239] // left value = acc[236+0:239+0] -v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] -v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[244:247] // left value = acc[244+0:247+0] -v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[248:251] // left value = acc[248+0:251+0] -v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[252:255] // left value = acc[252+0:255+0] - -v_mov_b32 v[vgprValuA_X0_I0+0+0], v[vgprValuA_X0_I0+8+0] -v_mov_b32 v[vgprValuA_X0_I0+1+0], v[vgprValuA_X0_I0+9+0] -v_mov_b32 v[vgprValuA_X0_I0+2+0], v[vgprValuA_X0_I0+10+0] -v_mov_b32 v[vgprValuA_X0_I0+3+0], v[vgprValuA_X0_I0+11+0] -v_mov_b32 v[vgprValuA_X0_I0+4+0], v[vgprValuA_X0_I0+12+0] -v_mov_b32 v[vgprValuA_X0_I0+5+0], v[vgprValuA_X0_I0+13+0] -v_mov_b32 v[vgprValuA_X0_I0+6+0], v[vgprValuA_X0_I0+14+0] -v_mov_b32 v[vgprValuA_X0_I0+7+0], v[vgprValuA_X0_I0+15+0] -v_mov_b32 v[vgprValuA_X0_I0+8+0], v[vgprValuA_X2_I0+0+0] -v_mov_b32 v[vgprValuA_X0_I0+9+0], v[vgprValuA_X2_I0+1+0] -v_mov_b32 v[vgprValuA_X0_I0+10+0], v[vgprValuA_X2_I0+2+0] -v_mov_b32 v[vgprValuA_X0_I0+11+0], v[vgprValuA_X2_I0+3+0] -v_mov_b32 v[vgprValuA_X0_I0+12+0], v[vgprValuA_X2_I0+4+0] -v_mov_b32 v[vgprValuA_X0_I0+13+0], v[vgprValuA_X2_I0+5+0] -v_mov_b32 v[vgprValuA_X0_I0+14+0], v[vgprValuA_X2_I0+6+0] -v_mov_b32 v[vgprValuA_X0_I0+15+0], v[vgprValuA_X2_I0+7+0] -v_mov_b32 v[vgprValuA_X2_I0+0+0], v[vgprValuA_X2_I0+8+0] -v_mov_b32 v[vgprValuA_X2_I0+1+0], v[vgprValuA_X2_I0+9+0] -v_mov_b32 v[vgprValuA_X2_I0+2+0], v[vgprValuA_X2_I0+10+0] -v_mov_b32 v[vgprValuA_X2_I0+3+0], v[vgprValuA_X2_I0+11+0] -v_mov_b32 v[vgprValuA_X2_I0+4+0], v[vgprValuA_X2_I0+12+0] -v_mov_b32 v[vgprValuA_X2_I0+5+0], v[vgprValuA_X2_I0+13+0] -v_mov_b32 v[vgprValuA_X2_I0+6+0], v[vgprValuA_X2_I0+14+0] -v_mov_b32 v[vgprValuA_X2_I0+7+0], v[vgprValuA_X2_I0+15+0] - -/* closeLoop loopL finalLoop=1 tailLoop=1 */ -s_sub_i32 s[sgprLoopCounterL], s[sgprLoopCounterL], 0x10 // dec counterL (tailLoop) -s_add_u32 s[sgprOrigLoopCounter], s[sgprOrigLoopCounter], 0x10 // inc counterL -s_cmp_le_i32 s[sgprLoopCounterL], 0x0 // counterL<=0 -s_cbranch_scc0 label_TailLoopBeginL // restart LoopL -label_TailLoopEndL: -label_SkipTailLoopL: - -/* Tail: remove address/G2L [160...230) from pool */ -label_Summation_End_SB904UR36QKZ73J2_0: -/* endSummation: add vgpr [0...230) to pool */ -.set sgprWGM, UNDEF -.set sgprLoopCounterL, UNDEF -.set sgprOrigLoopCounter, UNDEF -.set sgprAddressA, UNDEF -.set sgprAddressB, UNDEF -.set sgprStridesA, UNDEF -.set sgprStridesB, UNDEF -.set sgprStaggerUIter, UNDEF -.set sgprSrdA, UNDEF -.set sgprSrdB, UNDEF -.set sgprShadowLimitA, UNDEF -.set sgprShadowLimitB, UNDEF -.set sgprWrapUA, UNDEF -.set sgprWrapUB, UNDEF -.set sgprGlobalReadIncsA, UNDEF -.set sgprGlobalReadIncsB, UNDEF -.set sgprScalarGlobalReadOffsetA, UNDEF -.set sgprScalarGlobalReadOffsetB, UNDEF -/* load store sgprs */ -.set sgprAddressScaleAlphaVec, 48 -.set sgprAddressBias, 50 -.set sgprBiasType, 52 -.set sgprBiasStride, 53 -.set sgpractivationAlpha, 54 -.set sgpractivationBeta, 55 -.set sgprActivationType, 56 -s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? -s_cbranch_scc0 label_GSU_4 // branch if GSU != 1 -/* Check if custom structure pointer is null */ -s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? -s_cbranch_scc1 label_LoadExternalEpilogueStruct_1 // branch if ArgType == 2 -s_load_dwordx8 s[48:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x58 -s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x78 -s_branch label_LoadExternalEpilogueStructEnd_1 -label_LoadExternalEpilogueStruct_1: -s_load_dwordx4 s[48:51], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x90 -s_load_dwordx2 s[52:53], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xa0 -s_load_dwordx2 s[54:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xb8 -s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0xc0 -label_LoadExternalEpilogueStructEnd_1: -label_GSU_4: -.set sgprSrdScaleAlphaVec, 32 -.set sgprSrdBias, 40 - -/* Mapping of Acc register -> C Vgpr register */ - -/* not-LocalSplitU: global write indices */ -/* computeStoreVgprs */ -v_lshrrev_b32 v4, 6, v[vgprSerial] // v4 = v[vgprSerial] / 64 -v_lshrrev_b32 v5, 2, v4 // v5 = v4 / 4 -v_mul_lo_u32 v5, 0x10, v5 // wave coordination offset 1 -v_and_b32 v1, 63, v[vgprSerial] // v1 = v[vgprSerial] % 64 -v_lshrrev_b32 v1, 4, v1 // v1 = v1 / 16 -v_lshlrev_b32 v1, 0x2, v1 // thread0 * continuous_output -v_add_lshl_u32 v1, v5, v1, 0 // coordination 1 = vwB *(wave_id1 + tid1) -v_mul_lo_u32 v2, v1, s[sgprStrideC1J] // offset 1 -v_mul_lo_u32 v3, v1, s[sgprStrideD1J] // offset 1 -v_and_b32 v0, 3, v4 // v0 = v4 % 4 -v_mul_lo_u32 v0, 0x10, v0 // wave coordination offset 0 -v_and_b32 v5, 15, v[vgprSerial] // v5 = v[vgprSerial] % 16 -v_add_lshl_u32 v0, v5, v0, 2 // coordination 0 = vwA * (wave_id0 + tid0) -s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_add_u32 v0, s8, v0 // coord 0 = (tid0/MI_m)*4 + waveG0*MIB_m + MT0*SG0 -s_mul_i32 s8, 256, s[sgprWorkGroup1] // wgp1 * MT1 -v_add_u32 v1, s8, v1 // coord 1 = (tid0%MI_m) + waveG1*MIB_n + MT1*SG1 - -/* not-LocalSplitU: global write */ - -/******************************************/ -/* Global Write Elements */ -/******************************************/ -s_waitcnt lgkmcnt(0) // wait for 36 bytes of kern args. -s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? -s_cbranch_scc1 label_GSU_5 // branch if GSU == 1 -s_and_b32 s58, 255, s[sgprSizeI] // s58 = s[sgprSizeI] % 256 -s_add_u32 s59, -0x1, s[sgprNumWorkGroups0] -s_cmp_ge_u32 s[sgprWorkGroup0], s59 // wg0 >= nwg0-1 ? -s_cselect_b32 s58, s58, 0 // set rMT0 -s_cmpk_gt_u32 s58, 0x0 // rMT0 > 0 -s_cbranch_scc1 label_GW_B0_E1_M // jump if edges required -s_and_b32 s58, 255, s[sgprSizeJ] // s58 = s[sgprSizeJ] % 256 -s_add_u32 s59, -0x1, s[sgprNumWorkGroups1] -s_cmp_ge_u32 s[sgprWorkGroup1], s59 // wg1 >= nwg1-1 -s_cselect_b32 s58, s58, 0 // set rMT1 -s_cmpk_gt_u32 s58, 0x0 // rMT1 > 0 -s_cbranch_scc1 label_GW_B0_E1_N // jump if edges required -label_GW_B0_E0_1: - -/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_lshl_u32 v10, v3, v0, 0x2 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 -v_accvgpr_read_b32 v[vgprValuC+12], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+13], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+14], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+15], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+16], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+17], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+18], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+19], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+20], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+21], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+22], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+23], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+24], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+25], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+26], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+27], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+28], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+29], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+30], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+31], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+32], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+33], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+34], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+35], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+36], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+37], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+38], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+39], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+40], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+41], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+42], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+43], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+44], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+45], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+46], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+47], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+48], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+49], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+50], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+51], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+52], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+53], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+54], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+55], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+56], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+57], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+58], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+59], acc47 // copy acc to vreg[47] -v_accvgpr_read_b32 v[vgprValuC+60], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+61], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+62], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+63], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+64], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+65], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+66], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+67], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+68], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+69], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+70], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+71], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+72], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+73], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+74], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+75], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+12], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+13], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+14], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+15], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+16], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+17], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+18], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+19], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+20], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+21], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+22], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+23], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+24], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+25], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+26], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+27], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+28], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+29], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+30], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+31], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+32], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+33], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+34], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+35], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+36], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+37], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+38], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+39], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+40], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+41], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+42], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+43], acc95 // copy acc to vreg[95] -v_accvgpr_read_b32 v[vgprValuC+44], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+45], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+46], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+47], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+48], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+49], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+50], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+51], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+52], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+53], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+54], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+55], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+56], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+57], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+58], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+59], acc111 // copy acc to vreg[111] -v_accvgpr_read_b32 v[vgprValuC+60], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+61], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+62], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+63], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+64], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+65], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+66], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+67], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+68], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+69], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+70], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+71], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+72], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+73], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+74], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+75], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+12], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+13], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+14], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+15], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+16], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+17], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+18], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+19], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+20], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+21], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+22], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+23], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+24], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+25], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+26], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+27], acc143 // copy acc to vreg[143] -v_accvgpr_read_b32 v[vgprValuC+28], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+29], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+30], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+31], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+32], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+33], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+34], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+35], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+36], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+37], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+38], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+39], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+40], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+41], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+42], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+43], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+44], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+45], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+46], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+47], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+48], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+49], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+50], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+51], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+52], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+53], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+54], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+55], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+56], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+57], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+58], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+59], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+60], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+61], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+62], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+63], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+64], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+65], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+66], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+67], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+68], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+69], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+70], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+71], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+72], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+73], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+74], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+75], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+12], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+13], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+14], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+15], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+16], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+17], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+18], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+19], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+20], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+21], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+22], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+23], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+24], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+25], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+26], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+27], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+28], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+29], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+30], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+31], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+32], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+33], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+34], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+35], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+36], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+37], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+38], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+39], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+40], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+41], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+42], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+43], acc223 // copy acc to vreg[223] -v_accvgpr_read_b32 v[vgprValuC+44], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+45], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+46], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+47], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+48], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+49], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+50], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+51], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+52], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+53], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+54], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+55], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+56], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+57], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+58], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+59], acc239 // copy acc to vreg[239] -v_accvgpr_read_b32 v[vgprValuC+60], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+61], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+62], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+63], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+64], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+65], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+66], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+67], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+68], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+69], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+70], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+71], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+72], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+73], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+74], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+75], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End_1 // jump to end -label_GW_B0_E1_N: - -/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v82, BufferOOB -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+12], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+13], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+14], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+15], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+16], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+17], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+18], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+19], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+24], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+25], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+26], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+27], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+28], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+29], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+30], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+31], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+32], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+33], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+34], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+35], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+36], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+37], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+38], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+39], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+44], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+45], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+46], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+47], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+48], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+49], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+50], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+51], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+52], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+53], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+54], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+55], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+56], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+57], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+58], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+59], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+64], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+65], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+66], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+67], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+68], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+69], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+70], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+71], acc47 // copy acc to vreg[47] -v_accvgpr_read_b32 v[vgprValuC+72], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+73], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+74], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+75], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+76], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+77], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+78], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+79], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+84], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+85], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+86], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+87], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+88], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+89], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+90], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+91], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v82, BufferOOB -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+12], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+13], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+14], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+15], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+16], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+17], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+18], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+19], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+24], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+25], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+26], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+27], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+28], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+29], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+30], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+31], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+32], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+33], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+34], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+35], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+36], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+37], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+38], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+39], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+44], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+45], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+46], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+47], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+48], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+49], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+50], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+51], acc95 // copy acc to vreg[95] -v_accvgpr_read_b32 v[vgprValuC+52], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+53], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+54], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+55], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+56], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+57], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+58], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+59], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+64], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+65], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+66], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+67], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+68], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+69], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+70], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+71], acc111 // copy acc to vreg[111] -v_accvgpr_read_b32 v[vgprValuC+72], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+73], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+74], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+75], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+76], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+77], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+78], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+79], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+84], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+85], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+86], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+87], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+88], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+89], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+90], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+91], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v82, BufferOOB -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+12], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+13], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+14], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+15], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+16], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+17], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+18], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+19], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+24], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+25], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+26], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+27], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+28], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+29], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+30], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+31], acc143 // copy acc to vreg[143] -v_accvgpr_read_b32 v[vgprValuC+32], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+33], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+34], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+35], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+36], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+37], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+38], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+39], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+44], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+45], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+46], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+47], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+48], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+49], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+50], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+51], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+52], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+53], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+54], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+55], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+56], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+57], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+58], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+59], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+64], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+65], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+66], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+67], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+68], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+69], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+70], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+71], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+72], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+73], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+74], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+75], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+76], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+77], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+78], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+79], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+84], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+85], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+86], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+87], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+88], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+89], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+90], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+91], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v82, BufferOOB -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+12], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+13], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+14], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+15], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+16], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+17], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+18], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+19], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+24], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+25], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+26], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+27], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+28], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+29], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+30], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+31], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+32], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+33], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+34], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+35], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+36], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+37], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+38], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+39], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+44], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+45], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+46], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+47], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+48], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+49], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+50], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+51], acc223 // copy acc to vreg[223] -v_accvgpr_read_b32 v[vgprValuC+52], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+53], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+54], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+55], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+56], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+57], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+58], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+59], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+64], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+65], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+66], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+67], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+68], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+69], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+70], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+71], acc239 // copy acc to vreg[239] -v_accvgpr_read_b32 v[vgprValuC+72], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+73], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+74], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+75], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+76], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+77], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+78], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+79], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+84], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+85], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+86], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+87], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+88], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+89], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+90], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+91], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End_1 // jump to end -label_GW_B0_E1_M: - -/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw1); (0,0,0,1:vw1); (0,0,0,2:vw1); (0,0,0,3:vw1); (0,0,1,0:vw1); (0,0,1,1:vw1); (0,0,1,2:vw1); (0,0,1,3:vw1); (0,0,2,0:vw1); (0,0,2,1:vw1); (0,0,2,2:vw1); (0,0,2,3:vw1); (0,0,3,0:vw1); (0,0,3,1:vw1); (0,0,3,2:vw1); (0,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+13], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+15], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+17], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+19], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+21], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+23], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+25], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+27], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+29], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+31], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+33], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+35], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+37], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+39], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+41], acc15 // copy acc to vreg[15] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ -/* (1,0,0,0:vw1); (1,0,0,1:vw1); (1,0,0,2:vw1); (1,0,0,3:vw1); (1,0,1,0:vw1); (1,0,1,1:vw1); (1,0,1,2:vw1); (1,0,1,3:vw1); (1,0,2,0:vw1); (1,0,2,1:vw1); (1,0,2,2:vw1); (1,0,2,3:vw1); (1,0,3,0:vw1); (1,0,3,1:vw1); (1,0,3,2:vw1); (1,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+13], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+15], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+17], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+19], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+21], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+23], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+25], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+27], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+29], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+31], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+33], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+35], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+37], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+39], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+41], acc31 // copy acc to vreg[31] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ -/* (2,0,0,0:vw1); (2,0,0,1:vw1); (2,0,0,2:vw1); (2,0,0,3:vw1); (2,0,1,0:vw1); (2,0,1,1:vw1); (2,0,1,2:vw1); (2,0,1,3:vw1); (2,0,2,0:vw1); (2,0,2,1:vw1); (2,0,2,2:vw1); (2,0,2,3:vw1); (2,0,3,0:vw1); (2,0,3,1:vw1); (2,0,3,2:vw1); (2,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+13], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+15], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+17], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+19], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+21], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+23], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+25], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+27], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+29], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+31], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+33], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+35], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+37], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+39], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+41], acc47 // copy acc to vreg[47] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ -/* (3,0,0,0:vw1); (3,0,0,1:vw1); (3,0,0,2:vw1); (3,0,0,3:vw1); (3,0,1,0:vw1); (3,0,1,1:vw1); (3,0,1,2:vw1); (3,0,1,3:vw1); (3,0,2,0:vw1); (3,0,2,1:vw1); (3,0,2,2:vw1); (3,0,2,3:vw1); (3,0,3,0:vw1); (3,0,3,1:vw1); (3,0,3,2:vw1); (3,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+13], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+15], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+17], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+19], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+21], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+23], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+25], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+27], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+29], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+31], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+33], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+35], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+37], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+39], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+41], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw1); (4,0,0,1:vw1); (4,0,0,2:vw1); (4,0,0,3:vw1); (4,0,1,0:vw1); (4,0,1,1:vw1); (4,0,1,2:vw1); (4,0,1,3:vw1); (4,0,2,0:vw1); (4,0,2,1:vw1); (4,0,2,2:vw1); (4,0,2,3:vw1); (4,0,3,0:vw1); (4,0,3,1:vw1); (4,0,3,2:vw1); (4,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+13], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+15], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+17], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+19], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+21], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+23], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+25], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+27], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+29], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+31], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+33], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+35], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+37], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+39], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+41], acc79 // copy acc to vreg[79] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 0, 1), (4, 0, 0, 2), (4, 0, 0, 3), (4, 0, 1, 0), (4, 0, 1, 1), (4, 0, 1, 2), (4, 0, 1, 3), (4, 0, 2, 0), (4, 0, 2, 1), (4, 0, 2, 2), (4, 0, 2, 3), (4, 0, 3, 0), (4, 0, 3, 1), (4, 0, 3, 2), (4, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #5 (d1,d0,vc1,vc0) = */ -/* (5,0,0,0:vw1); (5,0,0,1:vw1); (5,0,0,2:vw1); (5,0,0,3:vw1); (5,0,1,0:vw1); (5,0,1,1:vw1); (5,0,1,2:vw1); (5,0,1,3:vw1); (5,0,2,0:vw1); (5,0,2,1:vw1); (5,0,2,2:vw1); (5,0,2,3:vw1); (5,0,3,0:vw1); (5,0,3,1:vw1); (5,0,3,2:vw1); (5,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+13], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+15], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+17], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+19], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+21], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+23], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+25], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+27], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+29], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+31], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+33], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+35], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+37], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+39], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+41], acc95 // copy acc to vreg[95] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(5, 0, 0, 0), (5, 0, 0, 1), (5, 0, 0, 2), (5, 0, 0, 3), (5, 0, 1, 0), (5, 0, 1, 1), (5, 0, 1, 2), (5, 0, 1, 3), (5, 0, 2, 0), (5, 0, 2, 1), (5, 0, 2, 2), (5, 0, 2, 3), (5, 0, 3, 0), (5, 0, 3, 1), (5, 0, 3, 2), (5, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #6 (d1,d0,vc1,vc0) = */ -/* (6,0,0,0:vw1); (6,0,0,1:vw1); (6,0,0,2:vw1); (6,0,0,3:vw1); (6,0,1,0:vw1); (6,0,1,1:vw1); (6,0,1,2:vw1); (6,0,1,3:vw1); (6,0,2,0:vw1); (6,0,2,1:vw1); (6,0,2,2:vw1); (6,0,2,3:vw1); (6,0,3,0:vw1); (6,0,3,1:vw1); (6,0,3,2:vw1); (6,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+13], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+15], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+17], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+19], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+21], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+23], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+25], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+27], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+29], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+31], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+33], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+35], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+37], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+39], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+41], acc111 // copy acc to vreg[111] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 0, 1), (6, 0, 0, 2), (6, 0, 0, 3), (6, 0, 1, 0), (6, 0, 1, 1), (6, 0, 1, 2), (6, 0, 1, 3), (6, 0, 2, 0), (6, 0, 2, 1), (6, 0, 2, 2), (6, 0, 2, 3), (6, 0, 3, 0), (6, 0, 3, 1), (6, 0, 3, 2), (6, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #7 (d1,d0,vc1,vc0) = */ -/* (7,0,0,0:vw1); (7,0,0,1:vw1); (7,0,0,2:vw1); (7,0,0,3:vw1); (7,0,1,0:vw1); (7,0,1,1:vw1); (7,0,1,2:vw1); (7,0,1,3:vw1); (7,0,2,0:vw1); (7,0,2,1:vw1); (7,0,2,2:vw1); (7,0,2,3:vw1); (7,0,3,0:vw1); (7,0,3,1:vw1); (7,0,3,2:vw1); (7,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+13], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+15], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+17], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+19], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+21], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+23], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+25], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+27], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+29], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+31], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+33], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+35], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+37], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+39], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+41], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 0, 1), (7, 0, 0, 2), (7, 0, 0, 3), (7, 0, 1, 0), (7, 0, 1, 1), (7, 0, 1, 2), (7, 0, 1, 3), (7, 0, 2, 0), (7, 0, 2, 1), (7, 0, 2, 2), (7, 0, 2, 3), (7, 0, 3, 0), (7, 0, 3, 1), (7, 0, 3, 2), (7, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #8 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw1); (8,0,0,1:vw1); (8,0,0,2:vw1); (8,0,0,3:vw1); (8,0,1,0:vw1); (8,0,1,1:vw1); (8,0,1,2:vw1); (8,0,1,3:vw1); (8,0,2,0:vw1); (8,0,2,1:vw1); (8,0,2,2:vw1); (8,0,2,3:vw1); (8,0,3,0:vw1); (8,0,3,1:vw1); (8,0,3,2:vw1); (8,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+13], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+15], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+17], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+19], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+21], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+23], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+25], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+27], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+29], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+31], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+33], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+35], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+37], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+39], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+41], acc143 // copy acc to vreg[143] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 0, 1), (8, 0, 0, 2), (8, 0, 0, 3), (8, 0, 1, 0), (8, 0, 1, 1), (8, 0, 1, 2), (8, 0, 1, 3), (8, 0, 2, 0), (8, 0, 2, 1), (8, 0, 2, 2), (8, 0, 2, 3), (8, 0, 3, 0), (8, 0, 3, 1), (8, 0, 3, 2), (8, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #9 (d1,d0,vc1,vc0) = */ -/* (9,0,0,0:vw1); (9,0,0,1:vw1); (9,0,0,2:vw1); (9,0,0,3:vw1); (9,0,1,0:vw1); (9,0,1,1:vw1); (9,0,1,2:vw1); (9,0,1,3:vw1); (9,0,2,0:vw1); (9,0,2,1:vw1); (9,0,2,2:vw1); (9,0,2,3:vw1); (9,0,3,0:vw1); (9,0,3,1:vw1); (9,0,3,2:vw1); (9,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+13], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+15], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+17], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+19], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+21], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+23], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+25], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+27], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+29], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+31], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+33], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+35], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+37], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+39], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+41], acc159 // copy acc to vreg[159] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 0, 1), (9, 0, 0, 2), (9, 0, 0, 3), (9, 0, 1, 0), (9, 0, 1, 1), (9, 0, 1, 2), (9, 0, 1, 3), (9, 0, 2, 0), (9, 0, 2, 1), (9, 0, 2, 2), (9, 0, 2, 3), (9, 0, 3, 0), (9, 0, 3, 1), (9, 0, 3, 2), (9, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #10 (d1,d0,vc1,vc0) = */ -/* (10,0,0,0:vw1); (10,0,0,1:vw1); (10,0,0,2:vw1); (10,0,0,3:vw1); (10,0,1,0:vw1); (10,0,1,1:vw1); (10,0,1,2:vw1); (10,0,1,3:vw1); (10,0,2,0:vw1); (10,0,2,1:vw1); (10,0,2,2:vw1); (10,0,2,3:vw1); (10,0,3,0:vw1); (10,0,3,1:vw1); (10,0,3,2:vw1); (10,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+13], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+15], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+17], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+19], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+21], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+23], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+25], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+27], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+29], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+31], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+33], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+35], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+37], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+39], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+41], acc175 // copy acc to vreg[175] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(10, 0, 0, 0), (10, 0, 0, 1), (10, 0, 0, 2), (10, 0, 0, 3), (10, 0, 1, 0), (10, 0, 1, 1), (10, 0, 1, 2), (10, 0, 1, 3), (10, 0, 2, 0), (10, 0, 2, 1), (10, 0, 2, 2), (10, 0, 2, 3), (10, 0, 3, 0), (10, 0, 3, 1), (10, 0, 3, 2), (10, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #11 (d1,d0,vc1,vc0) = */ -/* (11,0,0,0:vw1); (11,0,0,1:vw1); (11,0,0,2:vw1); (11,0,0,3:vw1); (11,0,1,0:vw1); (11,0,1,1:vw1); (11,0,1,2:vw1); (11,0,1,3:vw1); (11,0,2,0:vw1); (11,0,2,1:vw1); (11,0,2,2:vw1); (11,0,2,3:vw1); (11,0,3,0:vw1); (11,0,3,1:vw1); (11,0,3,2:vw1); (11,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+13], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+15], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+17], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+19], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+21], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+23], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+25], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+27], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+29], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+31], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+33], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+35], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+37], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+39], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+41], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(11, 0, 0, 0), (11, 0, 0, 1), (11, 0, 0, 2), (11, 0, 0, 3), (11, 0, 1, 0), (11, 0, 1, 1), (11, 0, 1, 2), (11, 0, 1, 3), (11, 0, 2, 0), (11, 0, 2, 1), (11, 0, 2, 2), (11, 0, 2, 3), (11, 0, 3, 0), (11, 0, 3, 1), (11, 0, 3, 2), (11, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #12 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw1); (12,0,0,1:vw1); (12,0,0,2:vw1); (12,0,0,3:vw1); (12,0,1,0:vw1); (12,0,1,1:vw1); (12,0,1,2:vw1); (12,0,1,3:vw1); (12,0,2,0:vw1); (12,0,2,1:vw1); (12,0,2,2:vw1); (12,0,2,3:vw1); (12,0,3,0:vw1); (12,0,3,1:vw1); (12,0,3,2:vw1); (12,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+13], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+15], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+17], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+19], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+21], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+23], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+25], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+27], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+29], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+31], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+33], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+35], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+37], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+39], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+41], acc207 // copy acc to vreg[207] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 0, 1), (12, 0, 0, 2), (12, 0, 0, 3), (12, 0, 1, 0), (12, 0, 1, 1), (12, 0, 1, 2), (12, 0, 1, 3), (12, 0, 2, 0), (12, 0, 2, 1), (12, 0, 2, 2), (12, 0, 2, 3), (12, 0, 3, 0), (12, 0, 3, 1), (12, 0, 3, 2), (12, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #13 (d1,d0,vc1,vc0) = */ -/* (13,0,0,0:vw1); (13,0,0,1:vw1); (13,0,0,2:vw1); (13,0,0,3:vw1); (13,0,1,0:vw1); (13,0,1,1:vw1); (13,0,1,2:vw1); (13,0,1,3:vw1); (13,0,2,0:vw1); (13,0,2,1:vw1); (13,0,2,2:vw1); (13,0,2,3:vw1); (13,0,3,0:vw1); (13,0,3,1:vw1); (13,0,3,2:vw1); (13,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+13], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+15], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+17], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+19], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+21], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+23], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+25], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+27], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+29], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+31], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+33], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+35], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+37], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+39], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+41], acc223 // copy acc to vreg[223] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(13, 0, 0, 0), (13, 0, 0, 1), (13, 0, 0, 2), (13, 0, 0, 3), (13, 0, 1, 0), (13, 0, 1, 1), (13, 0, 1, 2), (13, 0, 1, 3), (13, 0, 2, 0), (13, 0, 2, 1), (13, 0, 2, 2), (13, 0, 2, 3), (13, 0, 3, 0), (13, 0, 3, 1), (13, 0, 3, 2), (13, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #14 (d1,d0,vc1,vc0) = */ -/* (14,0,0,0:vw1); (14,0,0,1:vw1); (14,0,0,2:vw1); (14,0,0,3:vw1); (14,0,1,0:vw1); (14,0,1,1:vw1); (14,0,1,2:vw1); (14,0,1,3:vw1); (14,0,2,0:vw1); (14,0,2,1:vw1); (14,0,2,2:vw1); (14,0,2,3:vw1); (14,0,3,0:vw1); (14,0,3,1:vw1); (14,0,3,2:vw1); (14,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+13], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+15], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+17], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+19], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+21], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+23], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+25], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+27], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+29], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+31], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+33], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+35], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+37], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+39], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+41], acc239 // copy acc to vreg[239] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 0, 1), (14, 0, 0, 2), (14, 0, 0, 3), (14, 0, 1, 0), (14, 0, 1, 1), (14, 0, 1, 2), (14, 0, 1, 3), (14, 0, 2, 0), (14, 0, 2, 1), (14, 0, 2, 2), (14, 0, 2, 3), (14, 0, 3, 0), (14, 0, 3, 1), (14, 0, 3, 2), (14, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #15 (d1,d0,vc1,vc0) = */ -/* (15,0,0,0:vw1); (15,0,0,1:vw1); (15,0,0,2:vw1); (15,0,0,3:vw1); (15,0,1,0:vw1); (15,0,1,1:vw1); (15,0,1,2:vw1); (15,0,1,3:vw1); (15,0,2,0:vw1); (15,0,2,1:vw1); (15,0,2,2:vw1); (15,0,2,3:vw1); (15,0,3,0:vw1); (15,0,3,1:vw1); (15,0,3,2:vw1); (15,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v42, BufferOOB -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 -v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+11], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+13], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+15], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+17], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+19], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+21], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+23], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+25], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+27], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+29], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+31], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+33], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+35], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+37], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+39], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+41], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 0, 1), (15, 0, 0, 2), (15, 0, 0, 3), (15, 0, 1, 0), (15, 0, 1, 1), (15, 0, 1, 2), (15, 0, 1, 3), (15, 0, 2, 0), (15, 0, 2, 1), (15, 0, 2, 2), (15, 0, 2, 3), (15, 0, 3, 0), (15, 0, 3, 1), (15, 0, 3, 2), (15, 0, 3, 3)] */ - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v8, 0x7fff0000 // fp32 Nan -v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 -buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End_1 // jump to end -label_GW_End_1: -s_getpc_b64 s[58:59] // addr of next instr -s_add_i32 s60, label_KernelEnd, 0x4 // target branch offset -s_add_u32 s58, s58, s60 // add target branch offset -s_addc_u32 s59, s59, 0 // add high and carry -s_setpc_b64 s[58:59] // branch to label_KernelEnd -label_GSU_5: -s_mov_b32 s[sgprSrdScaleAlphaVec+0], s[sgprAddressScaleAlphaVec+0] // init SRD base address (lower) -s_mov_b32 s[sgprSrdScaleAlphaVec+1], s[sgprAddressScaleAlphaVec+1] // init SRD base address (upper) + other fields -s_mov_b32 s[sgprSrdScaleAlphaVec+3], Srd127_96 // Set bits 127_96 in post-loop SRD -s_cmp_eq_u64 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], 0 // s[AddressScaleAlphaVec] == 0 ? -s_cbranch_scc0 label_ScaleAlphaVec_1AddrValid // branch if s[AddressScaleAlphaVec] != 0 -s_mov_b32 s[sgprSrdScaleAlphaVec+2], 0 -s_branch label_ScaleAlphaVec_1AddrValid_End -label_ScaleAlphaVec_1AddrValid: -s_mov_b32 s[sgprSrdScaleAlphaVec+2], s[sgprSizeI] -label_ScaleAlphaVec_1AddrValid_End: - -s_mul_i32 s[sgprSrdScaleAlphaVec+2], 0x4, s[sgprSrdScaleAlphaVec+2] // ScaleAlphaVec scaled by BPE -s_add_u32 s8, s[sgprWorkGroup2], 0x1 -s_mul_i32 s8, s[sgprBiasStride], s8 // stride * (wg+1) -s_cmp_eq_u32 s8, 0x0 // bias stride = 0? -s_cselect_b32 s8, s[sgprSizeI], s8 -s_mov_b32 s[sgprSrdBias+0], s[sgprAddressBias+0] // init SRD base address (lower) -s_mov_b32 s[sgprSrdBias+1], s[sgprAddressBias+1] // init SRD base address (upper) + other fields -s_mov_b32 s[sgprSrdBias+3], Srd127_96 // Set bits 127_96 in post-loop SRD -s_cmp_eq_u64 s[sgprAddressBias:sgprAddressBias+1], 0 // s[AddressBias] == 0 ? -s_cbranch_scc0 label_Bias_1AddrValid // branch if s[AddressBias] != 0 -s_mov_b32 s[sgprSrdBias+2], 0 -s_branch label_Bias_1AddrValid_End -label_Bias_1AddrValid: -s_mov_b32 s[sgprSrdBias+2], s8 -label_Bias_1AddrValid_End: - -label_Load_Biasf32_0_1: -s_cmpk_lg_u32 s[sgprBiasType], 0 // BiasType != 0 -s_cbranch_scc1 label_Load_Biasbf16_0_1 // Branch if true - -/******************************************/ -/* Read Bias to LDS */ -/******************************************/ -s_mul_i32 s[sgprSrdBias+2], 0x4, s[sgprSrdBias+2] // scaled by BPE -s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset -s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG -v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG -v_lshlrev_b32 v8, 0x2, v8 // Global bias address scaled by BPE -buffer_load_dword v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias -v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE -s_waitcnt vmcnt(0) // wait for bias load -s_barrier // Wait for all wavefronts -ds_write_b32 v8, v4 offset:0 // store bias -s_branch label_Load_Bias_End_1 // Branch to load bias end -label_Load_Biasbf16_0_1: -s_cmpk_lg_u32 s[sgprBiasType], 7 // BiasType != 7 -s_cbranch_scc1 label_Load_Bias_End_1 // Branch if true - -/******************************************/ -/* Read Bias to LDS */ -/******************************************/ -s_mul_i32 s[sgprSrdBias+2], 0x2, s[sgprSrdBias+2] // scaled by BPE -s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset -s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG -v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG -v_lshlrev_b32 v8, 0x1, v8 // Global bias address scaled by BPE -buffer_load_short_d16 v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias -v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE -s_waitcnt vmcnt(0) // wait for bias load -s_barrier // Wait for all wavefronts -v_lshlrev_b32 v4, 16, v4 // cvt bf16 to fp32. -ds_write_b32 v8, v4 offset:0 // store bias -s_branch label_Load_Bias_End_1 // Branch to load bias end -label_Load_Bias_End_1: -s_cmpk_eq_u32 s[sgprBeta], 0x0 // Beta == 0 -s_cbranch_scc0 label_GW_Beta_2 // Branch if Beta is not zero - -s_and_b32 s60, 255, s[sgprSizeI] // s60 = s[sgprSizeI] % 256 -s_add_u32 s61, -0x1, s[sgprNumWorkGroups0] -s_cmp_ge_u32 s[sgprWorkGroup0], s61 // wg0 >= nwg0-1 ? -s_cselect_b32 s60, s60, 0 // set rMT0 -s_cmpk_gt_u32 s60, 0x0 // rMT0 > 0 -s_cbranch_scc0 label_NoBranch_PFO42GJLMDBXSWVP_0 // Only branch on scc1 -// jump if edges required -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_B0_E1_M_1, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_B0_E1_M_1 -label_NoBranch_PFO42GJLMDBXSWVP_0: -s_and_b32 s60, 255, s[sgprSizeJ] // s60 = s[sgprSizeJ] % 256 -s_add_u32 s61, -0x1, s[sgprNumWorkGroups1] -s_cmp_ge_u32 s[sgprWorkGroup1], s61 // wg1 >= nwg1-1 -s_cselect_b32 s60, s60, 0 // set rMT1 -s_cmpk_gt_u32 s60, 0x0 // rMT1 > 0 -s_cbranch_scc0 label_NoBranch_XR5UBTMCR0HWLF5H_0 // Only branch on scc1 -// jump if edges required -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_B0_E1_N_1, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_B0_E1_N_1 -label_NoBranch_XR5UBTMCR0HWLF5H_0: -label_GW_B0_E0_2: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_0_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_0_edge_0 // Branch if true -label_To_Activation_None_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Abs_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Gelu_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Relu_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Sigmoid_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Tanh_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Geluscaling_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_To_Activation_Silu_VW4_1_beta_0_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_6 -label_ActivationSetPCAddrEnd_6: - -/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b128 v[20:23], v15 offset:0 // load bias -v_lshlrev_b32 v16, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_lshl_u32 v13, v3, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 -v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+32], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+33], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+34], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+35], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+36], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+37], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+38], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+39], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+40], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+41], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+42], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+43], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+44], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+45], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+46], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+47], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+48], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+49], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+50], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+51], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+52], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+53], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+54], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+55], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+56], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+57], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+58], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+59], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+60], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+61], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+62], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+63], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+64], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+65], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+66], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+67], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+68], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+69], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+70], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+71], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+72], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+73], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+74], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+75], acc47 // copy acc to vreg[47] -v_accvgpr_read_b32 v[vgprValuC+76], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+77], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+78], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+79], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+80], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+81], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+82], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+83], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+84], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+85], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+86], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+87], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+88], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+89], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+90], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+91], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+28], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+29], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+31], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+32], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+33], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+34], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+35], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+36], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+37], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+38], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+39], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+40], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+41], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+42], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+43], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+44], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+45], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+46], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+47], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+48], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+49], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+50], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+51], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+52], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+53], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+54], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+55], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+56], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+57], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+58], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+59], acc95 // copy acc to vreg[95] -v_accvgpr_read_b32 v[vgprValuC+60], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+61], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+62], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+63], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+64], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+65], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+66], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+67], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+68], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+69], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+70], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+71], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+72], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+73], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+74], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+75], acc111 // copy acc to vreg[111] -v_accvgpr_read_b32 v[vgprValuC+76], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+77], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+78], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+79], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+80], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+81], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+82], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+83], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+84], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+85], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+86], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+87], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+88], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+89], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+90], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+91], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+28], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+29], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+31], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+32], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+33], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+34], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+35], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+36], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+37], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+38], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+39], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+40], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+41], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+42], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+43], acc143 // copy acc to vreg[143] -v_accvgpr_read_b32 v[vgprValuC+44], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+45], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+46], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+47], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+48], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+49], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+50], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+51], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+52], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+53], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+54], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+55], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+56], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+57], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+58], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+59], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+60], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+61], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+62], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+63], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+64], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+65], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+66], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+67], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+68], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+69], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+70], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+71], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+72], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+73], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+74], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+75], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+76], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+77], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+78], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+79], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+80], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+81], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+82], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+83], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+84], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+85], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+86], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+87], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+88], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+89], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+90], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+91], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+32], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+33], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+34], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+35], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+36], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+37], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+38], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+39], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+40], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+41], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+42], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+43], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+44], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+45], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+46], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+47], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+48], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+49], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+50], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+51], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+52], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+53], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+54], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+55], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+56], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+57], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+58], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+59], acc223 // copy acc to vreg[223] -v_accvgpr_read_b32 v[vgprValuC+60], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+61], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+62], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+63], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+64], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+65], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+66], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+67], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+68], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+69], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+70], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+71], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+72], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+73], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+74], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+75], acc239 // copy acc to vreg[239] -v_accvgpr_read_b32 v[vgprValuC+76], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+77], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+78], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+79], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+80], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+81], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+82], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+83], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+84], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+85], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+86], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+87], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+88], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+89], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+90], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+91], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -// jump to end -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_End_2, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_End_2 -label_GW_B0_E1_N_1: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_0_edge_1 // Branch if true -label_To_Activation_None_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Abs_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Gelu_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Relu_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Sigmoid_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Tanh_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Geluscaling_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_To_Activation_Silu_VW4_1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_5 -label_ActivationSetPCAddrEnd_5: - -/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=14 */ -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v115, BufferOOB -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b128 v[16:19], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v29, v0, s60 -v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE -v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v36, v0, s60 -v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE -v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v39, v0, s60 -v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE -v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v0, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v0, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v64, v0, s60 -v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE -v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v0, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v0, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v0, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v95, v0, s60 -v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE -v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v102, v0, s60 -v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE -v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v113, v0, s60 -v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE -v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+24], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+25], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+26], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+27], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+32], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+33], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+34], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+35], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+40], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+41], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+42], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+43], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+48], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+49], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+50], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+51], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+52], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+53], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+54], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+55], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+60], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+61], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+62], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+63], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+68], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+69], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+70], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+71], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+76], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+77], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+78], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+79], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+80], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+81], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+82], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+83], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+88], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+89], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+90], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+91], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+96], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+97], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+98], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+99], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+104], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+105], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+106], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+107], acc47 // copy acc to vreg[47] -v_accvgpr_read_b32 v[vgprValuC+108], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+109], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+110], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+111], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+116], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+117], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+118], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+119], acc61 // copy acc to vreg[55] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0)] */ -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha -v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha -v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha -v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_mov_b32 v25, v5 -v_mov_b32 v26, v6 -v_mov_b32 v27, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan -v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] -v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan -v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] -v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword -buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v104, v4 -v_mov_b32 v105, v5 -v_mov_b32 v106, v6 -v_mov_b32 v107, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan -v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan -v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] -v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan -v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan -v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] -v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword -buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ -/* (3,0,2,0:vw4); (3,0,3,0:vw4); (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v115, BufferOOB -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[16:19], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v29, v0, s60 -v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE -v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v36, v0, s60 -v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE -v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v39, v0, s60 -v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE -v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v0, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v0, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v64, v0, s60 -v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE -v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v0, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v0, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v0, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v95, v0, s60 -v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE -v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v102, v0, s60 -v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE -v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v113, v0, s60 -v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE -v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+24], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+25], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+26], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+27], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+32], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+33], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+34], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+35], acc63 // copy acc to vreg[63] -v_accvgpr_read_b32 v[vgprValuC+40], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+41], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+42], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+43], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+48], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+49], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+50], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+51], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+52], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+53], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+54], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+55], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+60], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+61], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+62], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+63], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+68], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+69], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+70], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+71], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+76], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+77], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+78], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+79], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+80], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+81], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+82], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+83], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+88], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+89], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+90], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+91], acc95 // copy acc to vreg[95] -v_accvgpr_read_b32 v[vgprValuC+96], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+97], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+98], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+99], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+104], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+105], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+106], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+107], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+108], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+109], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+110], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+111], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+116], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+117], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+118], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+119], acc111 // copy acc to vreg[111] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(3, 0, 2, 0), (3, 0, 3, 0), (4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha -v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha -v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha -v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_mov_b32 v25, v5 -v_mov_b32 v26, v6 -v_mov_b32 v27, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan -v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] -v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan -v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] -v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword -buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v104, v4 -v_mov_b32 v105, v5 -v_mov_b32 v106, v6 -v_mov_b32 v107, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan -v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan -v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] -v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan -v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan -v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] -v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword -buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ -/* (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4); (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v115, BufferOOB -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[16:19], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v29, v0, s60 -v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE -v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v36, v0, s60 -v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE -v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v39, v0, s60 -v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE -v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v0, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v0, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v64, v0, s60 -v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE -v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v0, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v0, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v0, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v95, v0, s60 -v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE -v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v102, v0, s60 -v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE -v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v113, v0, s60 -v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE -v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+24], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+25], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+26], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+27], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+32], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+33], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+34], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+35], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+40], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+41], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+42], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+43], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+48], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+49], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+50], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+51], acc127 // copy acc to vreg[127] -v_accvgpr_read_b32 v[vgprValuC+52], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+53], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+54], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+55], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+60], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+61], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+62], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+63], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+68], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+69], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+70], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+71], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+76], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+77], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+78], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+79], acc143 // copy acc to vreg[143] -v_accvgpr_read_b32 v[vgprValuC+80], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+81], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+82], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+83], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+88], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+89], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+90], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+91], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+96], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+97], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+98], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+99], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+104], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+105], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+106], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+107], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+108], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+109], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+110], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+111], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+116], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+117], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+118], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+119], acc173 // copy acc to vreg[167] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0), (8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0)] */ -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha -v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha -v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha -v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_mov_b32 v25, v5 -v_mov_b32 v26, v6 -v_mov_b32 v27, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan -v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] -v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan -v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] -v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword -buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v104, v4 -v_mov_b32 v105, v5 -v_mov_b32 v106, v6 -v_mov_b32 v107, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan -v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan -v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] -v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan -v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan -v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] -v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword -buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ -/* (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4); (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v115, BufferOOB -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[16:19], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v29, v0, s60 -v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE -v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v36, v0, s60 -v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE -v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v39, v0, s60 -v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE -v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v0, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v0, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v64, v0, s60 -v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE -v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v0, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v0, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v0, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v95, v0, s60 -v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE -v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v102, v0, s60 -v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE -v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v113, v0, s60 -v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE -v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+24], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+25], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+26], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+27], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+32], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+33], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+34], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+35], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+40], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+41], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+42], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+43], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+48], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+49], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+50], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+51], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+52], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+53], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+54], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+55], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+60], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+61], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+62], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+63], acc191 // copy acc to vreg[191] -v_accvgpr_read_b32 v[vgprValuC+68], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+69], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+70], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+71], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+76], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+77], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+78], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+79], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+80], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+81], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+82], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+83], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+88], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+89], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+90], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+91], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+96], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+97], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+98], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+99], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+104], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+105], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+106], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+107], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+108], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+109], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+110], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+111], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+116], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+117], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+118], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+119], acc223 // copy acc to vreg[223] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0), (12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha -v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha -v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha -v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_mov_b32 v25, v5 -v_mov_b32 v26, v6 -v_mov_b32 v27, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan -v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] -v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan -v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] -v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword -buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v104, v4 -v_mov_b32 v105, v5 -v_mov_b32 v106, v6 -v_mov_b32 v107, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan -v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan -v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] -v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan -v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan -v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] -v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword -buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ -/* (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v73, BufferOOB -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v73, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[16:19], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v73, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v29, v0, s60 -v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE -v_cndmask_b32 v29, v73, v29, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v28, v73, v28, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v36, v0, s60 -v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE -v_cndmask_b32 v36, v73, v36, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v73, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v39, v0, s60 -v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE -v_cndmask_b32 v39, v73, v39, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v38, v73, v38, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v0, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v73, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v73, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v0, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v73, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v73, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v64, v0, s60 -v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE -v_cndmask_b32 v64, v73, v64, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v59, v73, v59, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v0, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v73, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v73, v66, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+24], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+25], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+26], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+27], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+32], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+33], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+34], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+35], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+40], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+41], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+42], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+43], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+48], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+49], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+50], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+51], acc239 // copy acc to vreg[239] -v_accvgpr_read_b32 v[vgprValuC+52], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+53], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+54], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+55], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+60], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+61], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+62], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+63], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+68], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+69], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+70], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+71], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+76], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+77], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+78], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+79], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha -v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha -v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_mov_b32 v25, v5 -v_mov_b32 v26, v6 -v_mov_b32 v27, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan -v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] -v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan -v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] -v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword -buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v32, v4 -v_mov_b32 v33, v5 -v_mov_b32 v34, v6 -v_mov_b32 v35, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan -v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan -v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan -v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] -v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword -buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_mov_b32 v69, v5 -v_mov_b32 v70, v6 -v_mov_b32 v71, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan -v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] -v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan -v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] -v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword -buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) -v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -// jump to end -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_End_2, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_End_2 -label_GW_B0_E1_M_1: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW1_beta_0_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW1_beta_0_edge_1 // Branch if true -label_To_Activation_None_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Abs_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Clippedrelu_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Gelu_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Leakyrelu_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Relu_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Sigmoid_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Tanh_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Geluscaling_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_To_Activation_Silu_VW1_beta_0_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_4 -label_ActivationSetPCAddrEnd_4: - -/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw1); (0,0,0,1:vw1); (0,0,0,2:vw1); (0,0,0,3:vw1); (0,0,1,0:vw1); (0,0,1,1:vw1); (0,0,1,2:vw1); (0,0,1,3:vw1); (0,0,2,0:vw1); (0,0,2,1:vw1); (0,0,2,2:vw1); (0,0,2,3:vw1); (0,0,3,0:vw1); (0,0,3,1:vw1); (0,0,3,2:vw1); (0,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+24], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+36], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+40], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+44], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+48], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+52], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+56], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+60], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+64], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+68], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+72], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+76], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+80], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+84], acc15 // copy acc to vreg[15] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ -/* (1,0,0,0:vw1); (1,0,0,1:vw1); (1,0,0,2:vw1); (1,0,0,3:vw1); (1,0,1,0:vw1); (1,0,1,1:vw1); (1,0,1,2:vw1); (1,0,1,3:vw1); (1,0,2,0:vw1); (1,0,2,1:vw1); (1,0,2,2:vw1); (1,0,2,3:vw1); (1,0,3,0:vw1); (1,0,3,1:vw1); (1,0,3,2:vw1); (1,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+24], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+30], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+36], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+40], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+44], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+48], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+52], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+56], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+60], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+64], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+68], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+72], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+76], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+80], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+84], acc31 // copy acc to vreg[31] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ -/* (2,0,0,0:vw1); (2,0,0,1:vw1); (2,0,0,2:vw1); (2,0,0,3:vw1); (2,0,1,0:vw1); (2,0,1,1:vw1); (2,0,1,2:vw1); (2,0,1,3:vw1); (2,0,2,0:vw1); (2,0,2,1:vw1); (2,0,2,2:vw1); (2,0,2,3:vw1); (2,0,3,0:vw1); (2,0,3,1:vw1); (2,0,3,2:vw1); (2,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+24], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+30], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+36], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+40], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+44], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+48], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+52], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+56], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+60], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+64], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+68], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+72], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+76], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+80], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+84], acc47 // copy acc to vreg[47] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ -/* (3,0,0,0:vw1); (3,0,0,1:vw1); (3,0,0,2:vw1); (3,0,0,3:vw1); (3,0,1,0:vw1); (3,0,1,1:vw1); (3,0,1,2:vw1); (3,0,1,3:vw1); (3,0,2,0:vw1); (3,0,2,1:vw1); (3,0,2,2:vw1); (3,0,2,3:vw1); (3,0,3,0:vw1); (3,0,3,1:vw1); (3,0,3,2:vw1); (3,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+24], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+30], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+36], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+40], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+44], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+48], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+52], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+56], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+60], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+64], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+68], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+72], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+76], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+80], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+84], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw1); (4,0,0,1:vw1); (4,0,0,2:vw1); (4,0,0,3:vw1); (4,0,1,0:vw1); (4,0,1,1:vw1); (4,0,1,2:vw1); (4,0,1,3:vw1); (4,0,2,0:vw1); (4,0,2,1:vw1); (4,0,2,2:vw1); (4,0,2,3:vw1); (4,0,3,0:vw1); (4,0,3,1:vw1); (4,0,3,2:vw1); (4,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+24], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+36], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+40], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+44], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+48], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+52], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+56], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+60], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+64], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+68], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+72], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+76], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+80], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+84], acc79 // copy acc to vreg[79] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 0, 1), (4, 0, 0, 2), (4, 0, 0, 3), (4, 0, 1, 0), (4, 0, 1, 1), (4, 0, 1, 2), (4, 0, 1, 3), (4, 0, 2, 0), (4, 0, 2, 1), (4, 0, 2, 2), (4, 0, 2, 3), (4, 0, 3, 0), (4, 0, 3, 1), (4, 0, 3, 2), (4, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #5 (d1,d0,vc1,vc0) = */ -/* (5,0,0,0:vw1); (5,0,0,1:vw1); (5,0,0,2:vw1); (5,0,0,3:vw1); (5,0,1,0:vw1); (5,0,1,1:vw1); (5,0,1,2:vw1); (5,0,1,3:vw1); (5,0,2,0:vw1); (5,0,2,1:vw1); (5,0,2,2:vw1); (5,0,2,3:vw1); (5,0,3,0:vw1); (5,0,3,1:vw1); (5,0,3,2:vw1); (5,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+24], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+30], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+36], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+40], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+44], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+48], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+52], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+56], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+60], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+64], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+68], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+72], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+76], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+80], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+84], acc95 // copy acc to vreg[95] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(5, 0, 0, 0), (5, 0, 0, 1), (5, 0, 0, 2), (5, 0, 0, 3), (5, 0, 1, 0), (5, 0, 1, 1), (5, 0, 1, 2), (5, 0, 1, 3), (5, 0, 2, 0), (5, 0, 2, 1), (5, 0, 2, 2), (5, 0, 2, 3), (5, 0, 3, 0), (5, 0, 3, 1), (5, 0, 3, 2), (5, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #6 (d1,d0,vc1,vc0) = */ -/* (6,0,0,0:vw1); (6,0,0,1:vw1); (6,0,0,2:vw1); (6,0,0,3:vw1); (6,0,1,0:vw1); (6,0,1,1:vw1); (6,0,1,2:vw1); (6,0,1,3:vw1); (6,0,2,0:vw1); (6,0,2,1:vw1); (6,0,2,2:vw1); (6,0,2,3:vw1); (6,0,3,0:vw1); (6,0,3,1:vw1); (6,0,3,2:vw1); (6,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+24], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+30], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+36], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+40], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+44], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+48], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+52], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+56], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+60], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+64], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+68], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+72], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+76], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+80], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+84], acc111 // copy acc to vreg[111] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 0, 1), (6, 0, 0, 2), (6, 0, 0, 3), (6, 0, 1, 0), (6, 0, 1, 1), (6, 0, 1, 2), (6, 0, 1, 3), (6, 0, 2, 0), (6, 0, 2, 1), (6, 0, 2, 2), (6, 0, 2, 3), (6, 0, 3, 0), (6, 0, 3, 1), (6, 0, 3, 2), (6, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #7 (d1,d0,vc1,vc0) = */ -/* (7,0,0,0:vw1); (7,0,0,1:vw1); (7,0,0,2:vw1); (7,0,0,3:vw1); (7,0,1,0:vw1); (7,0,1,1:vw1); (7,0,1,2:vw1); (7,0,1,3:vw1); (7,0,2,0:vw1); (7,0,2,1:vw1); (7,0,2,2:vw1); (7,0,2,3:vw1); (7,0,3,0:vw1); (7,0,3,1:vw1); (7,0,3,2:vw1); (7,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+24], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+30], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+36], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+40], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+44], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+48], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+52], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+56], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+60], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+64], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+68], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+72], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+76], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+80], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+84], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 0, 1), (7, 0, 0, 2), (7, 0, 0, 3), (7, 0, 1, 0), (7, 0, 1, 1), (7, 0, 1, 2), (7, 0, 1, 3), (7, 0, 2, 0), (7, 0, 2, 1), (7, 0, 2, 2), (7, 0, 2, 3), (7, 0, 3, 0), (7, 0, 3, 1), (7, 0, 3, 2), (7, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #8 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw1); (8,0,0,1:vw1); (8,0,0,2:vw1); (8,0,0,3:vw1); (8,0,1,0:vw1); (8,0,1,1:vw1); (8,0,1,2:vw1); (8,0,1,3:vw1); (8,0,2,0:vw1); (8,0,2,1:vw1); (8,0,2,2:vw1); (8,0,2,3:vw1); (8,0,3,0:vw1); (8,0,3,1:vw1); (8,0,3,2:vw1); (8,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+24], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+36], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+40], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+44], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+48], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+52], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+56], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+60], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+64], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+68], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+72], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+76], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+80], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+84], acc143 // copy acc to vreg[143] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 0, 1), (8, 0, 0, 2), (8, 0, 0, 3), (8, 0, 1, 0), (8, 0, 1, 1), (8, 0, 1, 2), (8, 0, 1, 3), (8, 0, 2, 0), (8, 0, 2, 1), (8, 0, 2, 2), (8, 0, 2, 3), (8, 0, 3, 0), (8, 0, 3, 1), (8, 0, 3, 2), (8, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #9 (d1,d0,vc1,vc0) = */ -/* (9,0,0,0:vw1); (9,0,0,1:vw1); (9,0,0,2:vw1); (9,0,0,3:vw1); (9,0,1,0:vw1); (9,0,1,1:vw1); (9,0,1,2:vw1); (9,0,1,3:vw1); (9,0,2,0:vw1); (9,0,2,1:vw1); (9,0,2,2:vw1); (9,0,2,3:vw1); (9,0,3,0:vw1); (9,0,3,1:vw1); (9,0,3,2:vw1); (9,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+24], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+30], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+36], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+40], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+44], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+48], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+52], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+56], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+60], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+64], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+68], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+72], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+76], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+80], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+84], acc159 // copy acc to vreg[159] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 0, 1), (9, 0, 0, 2), (9, 0, 0, 3), (9, 0, 1, 0), (9, 0, 1, 1), (9, 0, 1, 2), (9, 0, 1, 3), (9, 0, 2, 0), (9, 0, 2, 1), (9, 0, 2, 2), (9, 0, 2, 3), (9, 0, 3, 0), (9, 0, 3, 1), (9, 0, 3, 2), (9, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #10 (d1,d0,vc1,vc0) = */ -/* (10,0,0,0:vw1); (10,0,0,1:vw1); (10,0,0,2:vw1); (10,0,0,3:vw1); (10,0,1,0:vw1); (10,0,1,1:vw1); (10,0,1,2:vw1); (10,0,1,3:vw1); (10,0,2,0:vw1); (10,0,2,1:vw1); (10,0,2,2:vw1); (10,0,2,3:vw1); (10,0,3,0:vw1); (10,0,3,1:vw1); (10,0,3,2:vw1); (10,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+24], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+30], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+36], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+40], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+44], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+48], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+52], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+56], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+60], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+64], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+68], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+72], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+76], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+80], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+84], acc175 // copy acc to vreg[175] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(10, 0, 0, 0), (10, 0, 0, 1), (10, 0, 0, 2), (10, 0, 0, 3), (10, 0, 1, 0), (10, 0, 1, 1), (10, 0, 1, 2), (10, 0, 1, 3), (10, 0, 2, 0), (10, 0, 2, 1), (10, 0, 2, 2), (10, 0, 2, 3), (10, 0, 3, 0), (10, 0, 3, 1), (10, 0, 3, 2), (10, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #11 (d1,d0,vc1,vc0) = */ -/* (11,0,0,0:vw1); (11,0,0,1:vw1); (11,0,0,2:vw1); (11,0,0,3:vw1); (11,0,1,0:vw1); (11,0,1,1:vw1); (11,0,1,2:vw1); (11,0,1,3:vw1); (11,0,2,0:vw1); (11,0,2,1:vw1); (11,0,2,2:vw1); (11,0,2,3:vw1); (11,0,3,0:vw1); (11,0,3,1:vw1); (11,0,3,2:vw1); (11,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+24], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+30], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+36], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+40], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+44], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+48], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+52], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+56], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+60], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+64], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+68], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+72], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+76], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+80], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+84], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(11, 0, 0, 0), (11, 0, 0, 1), (11, 0, 0, 2), (11, 0, 0, 3), (11, 0, 1, 0), (11, 0, 1, 1), (11, 0, 1, 2), (11, 0, 1, 3), (11, 0, 2, 0), (11, 0, 2, 1), (11, 0, 2, 2), (11, 0, 2, 3), (11, 0, 3, 0), (11, 0, 3, 1), (11, 0, 3, 2), (11, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #12 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw1); (12,0,0,1:vw1); (12,0,0,2:vw1); (12,0,0,3:vw1); (12,0,1,0:vw1); (12,0,1,1:vw1); (12,0,1,2:vw1); (12,0,1,3:vw1); (12,0,2,0:vw1); (12,0,2,1:vw1); (12,0,2,2:vw1); (12,0,2,3:vw1); (12,0,3,0:vw1); (12,0,3,1:vw1); (12,0,3,2:vw1); (12,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+24], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+36], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+40], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+44], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+48], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+52], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+56], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+60], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+64], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+68], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+72], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+76], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+80], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+84], acc207 // copy acc to vreg[207] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 0, 1), (12, 0, 0, 2), (12, 0, 0, 3), (12, 0, 1, 0), (12, 0, 1, 1), (12, 0, 1, 2), (12, 0, 1, 3), (12, 0, 2, 0), (12, 0, 2, 1), (12, 0, 2, 2), (12, 0, 2, 3), (12, 0, 3, 0), (12, 0, 3, 1), (12, 0, 3, 2), (12, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #13 (d1,d0,vc1,vc0) = */ -/* (13,0,0,0:vw1); (13,0,0,1:vw1); (13,0,0,2:vw1); (13,0,0,3:vw1); (13,0,1,0:vw1); (13,0,1,1:vw1); (13,0,1,2:vw1); (13,0,1,3:vw1); (13,0,2,0:vw1); (13,0,2,1:vw1); (13,0,2,2:vw1); (13,0,2,3:vw1); (13,0,3,0:vw1); (13,0,3,1:vw1); (13,0,3,2:vw1); (13,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+24], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+30], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+36], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+40], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+44], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+48], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+52], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+56], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+60], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+64], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+68], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+72], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+76], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+80], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+84], acc223 // copy acc to vreg[223] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(13, 0, 0, 0), (13, 0, 0, 1), (13, 0, 0, 2), (13, 0, 0, 3), (13, 0, 1, 0), (13, 0, 1, 1), (13, 0, 1, 2), (13, 0, 1, 3), (13, 0, 2, 0), (13, 0, 2, 1), (13, 0, 2, 2), (13, 0, 2, 3), (13, 0, 3, 0), (13, 0, 3, 1), (13, 0, 3, 2), (13, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #14 (d1,d0,vc1,vc0) = */ -/* (14,0,0,0:vw1); (14,0,0,1:vw1); (14,0,0,2:vw1); (14,0,0,3:vw1); (14,0,1,0:vw1); (14,0,1,1:vw1); (14,0,1,2:vw1); (14,0,1,3:vw1); (14,0,2,0:vw1); (14,0,2,1:vw1); (14,0,2,2:vw1); (14,0,2,3:vw1); (14,0,3,0:vw1); (14,0,3,1:vw1); (14,0,3,2:vw1); (14,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+24], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+30], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+36], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+40], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+44], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+48], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+52], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+56], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+60], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+64], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+68], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+72], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+76], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+80], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+84], acc239 // copy acc to vreg[239] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 0, 1), (14, 0, 0, 2), (14, 0, 0, 3), (14, 0, 1, 0), (14, 0, 1, 1), (14, 0, 1, 2), (14, 0, 1, 3), (14, 0, 2, 0), (14, 0, 2, 1), (14, 0, 2, 2), (14, 0, 2, 3), (14, 0, 3, 0), (14, 0, 3, 1), (14, 0, 3, 2), (14, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Edge Batch #15 (d1,d0,vc1,vc0) = */ -/* (15,0,0,0:vw1); (15,0,0,1:vw1); (15,0,0,2:vw1); (15,0,0,3:vw1); (15,0,1,0:vw1); (15,0,1,1:vw1); (15,0,1,2:vw1); (15,0,1,3:vw1); (15,0,2,0:vw1); (15,0,2,1:vw1); (15,0,2,2:vw1); (15,0,2,3:vw1); (15,0,3,0:vw1); (15,0,3,1:vw1); (15,0,3,2:vw1); (15,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v85, BufferOOB -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v16, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v20, v4, s60 -v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE -v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v22, v20 offset:0 // load bias -v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v26, v4, s60 -v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE -v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v28, v26 offset:0 // load bias -v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v32, v4, s60 -v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE -v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v34, v32 offset:0 // load bias -v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v38, v0, s60 -v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE -v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v4, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v46, v4, s60 -v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE -v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v50, v4, s60 -v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE -v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v58, v4, s60 -v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE -v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v4, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v66, v4, s60 -v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE -v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v70, v0, s60 -v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE -v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v74, v4, s60 -v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE -v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v78, v4, s60 -v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE -v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v4, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+18], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+24], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+30], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+36], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+40], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+44], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+48], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+52], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+56], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+60], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+64], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+68], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+72], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+76], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+80], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+84], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 0, 1), (15, 0, 0, 2), (15, 0, 0, 3), (15, 0, 1, 0), (15, 0, 1, 1), (15, 0, 1, 2), (15, 0, 1, 3), (15, 0, 2, 0), (15, 0, 2, 1), (15, 0, 2, 2), (15, 0, 2, 3), (15, 0, 3, 0), (15, 0, 3, 1), (15, 0, 3, 2), (15, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha -v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+18] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v18, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan -v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] -v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 -buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+24] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v24, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan -v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] -v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 -buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+30] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v30, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 -buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+36] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 -buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+44] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 -buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+48] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 -buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+52] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 -buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+56] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 -buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+64] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 -buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+68] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v68, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan -v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] -v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 -buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul -v_add_f32 v4, v16, v[vgprValuC+72] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 -buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul -v_add_f32 v4, v22, v[vgprValuC+76] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 -buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_add_f32 v4, v28, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul -v_add_f32 v4, v34, v[vgprValuC+84] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 -buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -// jump to end -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_End_2, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_End_2 -label_GW_Beta_2: -s_and_b32 s60, 255, s[sgprSizeI] // s60 = s[sgprSizeI] % 256 -s_add_u32 s61, -0x1, s[sgprNumWorkGroups0] -s_cmp_ge_u32 s[sgprWorkGroup0], s61 // wg0 >= nwg0-1 ? -s_cselect_b32 s60, s60, 0 // set rMT0 -s_cmpk_gt_u32 s60, 0x0 // rMT0 > 0 -s_cbranch_scc0 label_NoBranch_1L38YJQL3BUJ48XK_0 // Only branch on scc1 -// jump if edges required -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_B1_E1_M, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_B1_E1_M -label_NoBranch_1L38YJQL3BUJ48XK_0: -s_and_b32 s60, 255, s[sgprSizeJ] // s60 = s[sgprSizeJ] % 256 -s_add_u32 s61, -0x1, s[sgprNumWorkGroups1] -s_cmp_ge_u32 s[sgprWorkGroup1], s61 // wg1 >= nwg1-1 -s_cselect_b32 s60, s60, 0 // set rMT1 -s_cmpk_gt_u32 s60, 0x0 // rMT1 > 0 -s_cbranch_scc0 label_NoBranch_XMVL70A9XU3BIJFQ_0 // Only branch on scc1 -// jump if edges required -s_getpc_b64 s[60:61] // addr of next instr -s_add_i32 s62, label_GW_B1_E1_N, 0x4 // target branch offset -s_add_u32 s60, s60, s62 // add target branch offset -s_addc_u32 s61, s61, 0 // add high and carry -s_setpc_b64 s[60:61] // branch to label_GW_B1_E1_N -label_NoBranch_XMVL70A9XU3BIJFQ_0: -label_GW_B1_E0: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_1_edge_0 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_1_edge_0 // Branch if true -label_To_Activation_None_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Abs_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Gelu_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Relu_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Sigmoid_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Tanh_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Geluscaling_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_To_Activation_Silu_VW4_1_beta_1_edge_0: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_3 -label_ActivationSetPCAddrEnd_3: - -/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_add_lshl_u32 v14, v2, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 -buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b128 v[20:23], v15 offset:0 // load bias -v_lshlrev_b32 v16, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -v_add_lshl_u32 v13, v3, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 -v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+36], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+37], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+38], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+39], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+40], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+41], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+42], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+43], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+48], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+49], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+50], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+51], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+52], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+53], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+54], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+55], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+60], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+61], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+62], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+63], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+64], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+65], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+66], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+67], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+72], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+73], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+74], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+75], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+76], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+77], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+78], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+79], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+84], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+85], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+86], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+87], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+88], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+89], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+90], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+91], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+96], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+97], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+98], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+99], acc47 // copy acc to vreg[47] -v_accvgpr_read_b32 v[vgprValuC+100], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+101], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+102], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+103], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+108], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+109], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+110], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+111], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+112], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+113], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+114], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+115], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+120], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+121], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+122], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+123], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha -v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha -v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha -v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha -v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha -v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha -v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha -v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v18, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v19, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v32, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v33, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v44, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v45, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v46, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v47, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v56, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v57, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v58, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v59, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v68, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v69, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v80, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v81, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v82, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v83, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v92, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v93, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v94, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v95, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v104, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v105, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v112, v4 -v_mov_b32 v113, v5 -v_mov_b32 v114, v6 -v_mov_b32 v115, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan -v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan -v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] -v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan -v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan -v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] -v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v116, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v117, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v120, v4 -v_mov_b32 v121, v5 -v_mov_b32 v122, v6 -v_mov_b32 v123, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan -v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan -v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] -v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan -v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan -v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] -v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Batch #1 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -v_accvgpr_read_b32 v[vgprValuC+28], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+29], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+31], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+36], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+37], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+38], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+39], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+40], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+41], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+42], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+43], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+48], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+49], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+50], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+51], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+52], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+53], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+54], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+55], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+60], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+61], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+62], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+63], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+64], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+65], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+66], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+67], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+72], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+73], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+74], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+75], acc95 // copy acc to vreg[95] -v_accvgpr_read_b32 v[vgprValuC+76], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+77], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+78], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+79], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+84], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+85], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+86], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+87], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+88], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+89], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+90], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+91], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+96], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+97], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+98], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+99], acc111 // copy acc to vreg[111] -v_accvgpr_read_b32 v[vgprValuC+100], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+101], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+102], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+103], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+108], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+109], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+110], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+111], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+112], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+113], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+114], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+115], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+120], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+121], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+122], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+123], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha -v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha -v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha -v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha -v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha -v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha -v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha -v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v18, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v19, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v32, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v33, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v44, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v45, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v46, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v47, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v56, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v57, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v58, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v59, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v68, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v69, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v80, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v81, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v82, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v83, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v92, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v93, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v94, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v95, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v104, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v105, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v112, v4 -v_mov_b32 v113, v5 -v_mov_b32 v114, v6 -v_mov_b32 v115, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan -v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan -v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] -v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan -v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan -v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] -v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v116, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v117, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v120, v4 -v_mov_b32 v121, v5 -v_mov_b32 v122, v6 -v_mov_b32 v123, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan -v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan -v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] -v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan -v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan -v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] -v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Batch #2 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -v_accvgpr_read_b32 v[vgprValuC+28], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+29], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+31], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+36], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+37], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+38], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+39], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+40], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+41], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+42], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+43], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+48], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+49], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+50], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+51], acc143 // copy acc to vreg[143] -v_accvgpr_read_b32 v[vgprValuC+52], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+53], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+54], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+55], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+60], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+61], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+62], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+63], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+64], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+65], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+66], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+67], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+72], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+73], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+74], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+75], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+76], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+77], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+78], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+79], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+84], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+85], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+86], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+87], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+88], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+89], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+90], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+91], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+96], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+97], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+98], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+99], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+100], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+101], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+102], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+103], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+108], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+109], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+110], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+111], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+112], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+113], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+114], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+115], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+120], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+121], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+122], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+123], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha -v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha -v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha -v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha -v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha -v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha -v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha -v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v18, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v19, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v32, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v33, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v44, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v45, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v46, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v47, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v56, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v57, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v58, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v59, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v68, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v69, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v80, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v81, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v82, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v83, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v92, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v93, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v94, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v95, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v104, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v105, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v112, v4 -v_mov_b32 v113, v5 -v_mov_b32 v114, v6 -v_mov_b32 v115, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan -v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan -v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] -v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan -v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan -v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] -v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v116, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v117, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v120, v4 -v_mov_b32 v121, v5 -v_mov_b32 v122, v6 -v_mov_b32 v123, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan -v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan -v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] -v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan -v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan -v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] -v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Batch #3 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v15, v0, s60 -v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE -ds_read_b128 v[20:23], v15 offset:0 // load bias -buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+36], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+37], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+38], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+39], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+40], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+41], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+42], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+43], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+48], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+49], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+50], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+51], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+52], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+53], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+54], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+55], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+60], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+61], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+62], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+63], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+64], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+65], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+66], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+67], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+72], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+73], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+74], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+75], acc223 // copy acc to vreg[223] -v_accvgpr_read_b32 v[vgprValuC+76], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+77], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+78], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+79], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+84], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+85], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+86], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+87], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+88], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+89], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+90], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+91], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+96], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+97], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+98], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+99], acc239 // copy acc to vreg[239] -v_accvgpr_read_b32 v[vgprValuC+100], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+101], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+102], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+103], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+108], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+109], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+110], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+111], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+112], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+113], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+114], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+115], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+120], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+121], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+122], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+123], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha -v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha -v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha -v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha -v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha -v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha -v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha -v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha -v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha -v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha -v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha -v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha -v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha -v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha -v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha -v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha -v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha -v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha -v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha -v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha -v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha -v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha -v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha -v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha -v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha -v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha -v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha -v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 - -s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v18, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v19, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v32, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v33, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_mov_b32 v41, v5 -v_mov_b32 v42, v6 -v_mov_b32 v43, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan -v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] -v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan -v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan -v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] -v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v44, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v45, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v48, v4 -v_mov_b32 v49, v5 -v_mov_b32 v50, v6 -v_mov_b32 v51, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan -v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan -v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] -v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan -v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] -v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v46, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v47, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v52, v4 -v_mov_b32 v53, v5 -v_mov_b32 v54, v6 -v_mov_b32 v55, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan -v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan -v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] -v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan -v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v56, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v57, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_mov_b32 v61, v5 -v_mov_b32 v62, v6 -v_mov_b32 v63, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan -v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] -v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan -v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan -v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] -v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v58, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v59, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v68, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v69, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v76, v4 -v_mov_b32 v77, v5 -v_mov_b32 v78, v6 -v_mov_b32 v79, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan -v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan -v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] -v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan -v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan -v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] -v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v80, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v81, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v84, v4 -v_mov_b32 v85, v5 -v_mov_b32 v86, v6 -v_mov_b32 v87, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan -v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan -v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan -v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] -v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v82, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v83, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v88, v4 -v_mov_b32 v89, v5 -v_mov_b32 v90, v6 -v_mov_b32 v91, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan -v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan -v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] -v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan -v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] -v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v92, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v93, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v96, v4 -v_mov_b32 v97, v5 -v_mov_b32 v98, v6 -v_mov_b32 v99, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan -v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan -v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] -v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan -v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan -v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] -v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v94, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v95, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v104, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v105, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v112, v4 -v_mov_b32 v113, v5 -v_mov_b32 v114, v6 -v_mov_b32 v115, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan -v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan -v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] -v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan -v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan -v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] -v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D - -s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v116, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v117, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v120, v4 -v_mov_b32 v121, v5 -v_mov_b32 v122, v6 -v_mov_b32 v123, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan -v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan -v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] -v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan -v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan -v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] -v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword -s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE -s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) -s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) -buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End_2 // jump to end -label_GW_B1_E1_N: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_1_edge_1 // Branch if true -label_To_Activation_None_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Abs_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Gelu_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Relu_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Sigmoid_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Tanh_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Geluscaling_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_To_Activation_Silu_VW4_1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_2 -label_ActivationSetPCAddrEnd_2: - -/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=12 */ -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v123, BufferOOB -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b128 v[20:23], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v19, v0, s60 -v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE -v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v40, v0, s60 -v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE -v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v49, v0, s60 -v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE -v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v63, v0, s60 -v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE -v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v76, v0, s60 -v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE -v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v90, v0, s60 -v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE -v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v99, v0, s60 -v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE -v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v112, v0, s60 -v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE -v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v121, v0, s60 -v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE -v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+36], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+37], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+38], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+39], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+44], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+45], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+46], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+47], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+56], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+57], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+58], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+59], acc15 // copy acc to vreg[15] -v_accvgpr_read_b32 v[vgprValuC+64], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+65], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+66], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+67], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+72], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+73], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+74], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+75], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+80], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+81], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+82], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+83], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+92], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+93], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+94], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+95], acc31 // copy acc to vreg[31] -v_accvgpr_read_b32 v[vgprValuC+100], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+101], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+102], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+103], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+108], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+109], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+110], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+111], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+116], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+117], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+118], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+119], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+128], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+129], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+130], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+131], acc47 // copy acc to vreg[47] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha -v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha -v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha -v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha -v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha -v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v16, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v17, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v42, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v43, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v52, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v53, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v60, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v61, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v78, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v79, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v88, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v89, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v92, v4 -v_mov_b32 v93, v5 -v_mov_b32 v94, v6 -v_mov_b32 v95, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan -v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan -v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] -v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan -v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword -buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v96, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v97, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v114, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v115, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v124, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v125, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v128, v4 -v_mov_b32 v129, v5 -v_mov_b32 v130, v6 -v_mov_b32 v131, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan -v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan -v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] -v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan -v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan -v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] -v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword -buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #1 (d1,d0,vc1,vc0) = */ -/* (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4); (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v123, BufferOOB -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[20:23], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v19, v0, s60 -v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE -v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v40, v0, s60 -v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE -v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v49, v0, s60 -v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE -v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v63, v0, s60 -v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE -v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v76, v0, s60 -v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE -v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v90, v0, s60 -v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE -v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v99, v0, s60 -v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE -v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v112, v0, s60 -v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE -v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v121, v0, s60 -v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE -v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+28], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+29], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+30], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+31], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+36], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+37], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+38], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+39], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+44], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+45], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+46], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+47], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+56], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+57], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+58], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+59], acc63 // copy acc to vreg[63] -v_accvgpr_read_b32 v[vgprValuC+64], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+65], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+66], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+67], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+72], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+73], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+74], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+75], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+80], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+81], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+82], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+83], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+92], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+93], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+94], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+95], acc79 // copy acc to vreg[79] -v_accvgpr_read_b32 v[vgprValuC+100], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+101], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+102], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+103], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+108], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+109], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+110], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+111], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+116], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+117], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+118], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+119], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+128], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+129], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+130], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+131], acc95 // copy acc to vreg[95] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0), (4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha -v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha -v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha -v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha -v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha -v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v16, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v17, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v42, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v43, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v52, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v53, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v60, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v61, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v78, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v79, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v88, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v89, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v92, v4 -v_mov_b32 v93, v5 -v_mov_b32 v94, v6 -v_mov_b32 v95, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan -v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan -v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] -v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan -v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword -buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v96, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v97, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v114, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v115, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v124, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v125, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v128, v4 -v_mov_b32 v129, v5 -v_mov_b32 v130, v6 -v_mov_b32 v131, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan -v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan -v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] -v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan -v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan -v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] -v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword -buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #2 (d1,d0,vc1,vc0) = */ -/* (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4); (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v123, BufferOOB -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[20:23], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v19, v0, s60 -v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE -v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v40, v0, s60 -v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE -v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v49, v0, s60 -v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE -v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v63, v0, s60 -v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE -v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v76, v0, s60 -v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE -v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v90, v0, s60 -v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE -v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v99, v0, s60 -v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE -v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v112, v0, s60 -v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE -v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v121, v0, s60 -v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE -v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+28], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+29], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+30], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+31], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+36], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+37], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+38], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+39], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+44], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+45], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+46], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+47], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+56], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+57], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+58], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+59], acc111 // copy acc to vreg[111] -v_accvgpr_read_b32 v[vgprValuC+64], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+65], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+66], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+67], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+72], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+73], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+74], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+75], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+80], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+81], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+82], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+83], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+92], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+93], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+94], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+95], acc127 // copy acc to vreg[127] -v_accvgpr_read_b32 v[vgprValuC+100], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+101], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+102], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+103], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+108], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+109], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+110], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+111], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+116], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+117], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+118], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+119], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+128], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+129], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+130], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+131], acc143 // copy acc to vreg[143] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0), (8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha -v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha -v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha -v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha -v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha -v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v16, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v17, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v42, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v43, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v52, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v53, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v60, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v61, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v78, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v79, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v88, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v89, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v92, v4 -v_mov_b32 v93, v5 -v_mov_b32 v94, v6 -v_mov_b32 v95, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan -v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan -v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] -v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan -v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword -buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v96, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v97, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v114, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v115, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v124, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v125, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v128, v4 -v_mov_b32 v129, v5 -v_mov_b32 v130, v6 -v_mov_b32 v131, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan -v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan -v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] -v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan -v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan -v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] -v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword -buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #3 (d1,d0,vc1,vc0) = */ -/* (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v123, BufferOOB -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[20:23], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v19, v0, s60 -v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE -v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v40, v0, s60 -v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE -v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v49, v0, s60 -v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE -v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v63, v0, s60 -v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE -v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v76, v0, s60 -v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE -v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v90, v0, s60 -v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE -v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v99, v0, s60 -v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE -v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v112, v0, s60 -v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE -v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v121, v0, s60 -v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE -v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+28], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+29], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+30], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+31], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+36], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+37], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+38], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+39], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+44], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+45], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+46], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+47], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+56], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+57], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+58], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+59], acc159 // copy acc to vreg[159] -v_accvgpr_read_b32 v[vgprValuC+64], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+65], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+66], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+67], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+72], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+73], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+74], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+75], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+80], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+81], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+82], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+83], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+92], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+93], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+94], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+95], acc175 // copy acc to vreg[175] -v_accvgpr_read_b32 v[vgprValuC+100], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+101], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+102], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+103], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+108], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+109], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+110], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+111], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+116], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+117], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+118], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+119], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+128], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+129], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+130], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+131], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha -v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha -v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha -v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha -v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha -v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v16, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v17, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v42, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v43, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v52, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v53, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v60, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v61, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v78, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v79, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v88, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v89, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v92, v4 -v_mov_b32 v93, v5 -v_mov_b32 v94, v6 -v_mov_b32 v95, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan -v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan -v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] -v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan -v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword -buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v96, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v97, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v114, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v115, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v124, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v125, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v128, v4 -v_mov_b32 v129, v5 -v_mov_b32 v130, v6 -v_mov_b32 v131, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan -v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan -v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] -v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan -v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan -v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] -v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword -buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #4 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v123, BufferOOB -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[20:23], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v19, v0, s60 -v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE -v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v40, v0, s60 -v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE -v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v49, v0, s60 -v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE -v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v54, v0, s60 -v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE -v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v63, v0, s60 -v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE -v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v76, v0, s60 -v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE -v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v85, v0, s60 -v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE -v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v90, v0, s60 -v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE -v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v99, v0, s60 -v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE -v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v112, v0, s60 -v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE -v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v121, v0, s60 -v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE -v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+36], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+37], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+38], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+39], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+44], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+45], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+46], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+47], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+56], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+57], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+58], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+59], acc207 // copy acc to vreg[207] -v_accvgpr_read_b32 v[vgprValuC+64], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+65], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+66], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+67], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+72], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+73], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+74], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+75], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+80], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+81], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+82], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+83], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+92], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+93], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+94], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+95], acc223 // copy acc to vreg[223] -v_accvgpr_read_b32 v[vgprValuC+100], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+101], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+102], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+103], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+108], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+109], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+110], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+111], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+116], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+117], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+118], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+119], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+128], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+129], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+130], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+131], acc239 // copy acc to vreg[239] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha -v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha -v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha -v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha -v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha -v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha -v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha -v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha -v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha -v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha -v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha -v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha -v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha -v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha -v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha -v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha -v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha -v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha -v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha -v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha -v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha -v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha -v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha -v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v16, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v17, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v42, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v43, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v52, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v53, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v60, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v61, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v64, v4 -v_mov_b32 v65, v5 -v_mov_b32 v66, v6 -v_mov_b32 v67, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan -v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan -v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan -v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] -v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword -buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v70, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v71, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v72, v4 -v_mov_b32 v73, v5 -v_mov_b32 v74, v6 -v_mov_b32 v75, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan -v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan -v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] -v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan -v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword -buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v78, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v79, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_mov_b32 v81, v5 -v_mov_b32 v82, v6 -v_mov_b32 v83, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan -v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] -v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan -v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan -v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] -v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword -buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v88, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v89, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v92, v4 -v_mov_b32 v93, v5 -v_mov_b32 v94, v6 -v_mov_b32 v95, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan -v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan -v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] -v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan -v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword -buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v96, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v97, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_mov_b32 v101, v5 -v_mov_b32 v102, v6 -v_mov_b32 v103, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan -v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] -v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan -v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan -v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] -v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword -buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v106, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v107, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v108, v4 -v_mov_b32 v109, v5 -v_mov_b32 v110, v6 -v_mov_b32 v111, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan -v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan -v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] -v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan -v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan -v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] -v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword -buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v114, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v115, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v116, v4 -v_mov_b32 v117, v5 -v_mov_b32 v118, v6 -v_mov_b32 v119, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan -v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan -v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] -v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan -v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan -v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] -v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword -buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v124, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v125, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v128, v4 -v_mov_b32 v129, v5 -v_mov_b32 v130, v6 -v_mov_b32 v131, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan -v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan -v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] -v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan -v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan -v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] -v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword -buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #5 (d1,d0,vc1,vc0) = */ -/* (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v51, BufferOOB -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v51, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v51, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b128 v[20:23], v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v51, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v51, v18, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v19, v0, s60 -v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE -v_cndmask_b32 v19, v51, v19, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v18, v51, v18, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v51, v33, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v40, v0, s60 -v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE -v_cndmask_b32 v40, v51, v40, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v33, v51, v33, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v51, v48, s[64:65] // LDC clip if OOB. offset -buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v49, v0, s60 -v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE -v_cndmask_b32 v49, v51, v49, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v48, v51, v48, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+28], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+29], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+30], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+31], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+36], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+37], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+38], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+39], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+44], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+45], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+46], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+47], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+56], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+57], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+58], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+59], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ -v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha -v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha -v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha -v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha -v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha -v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha -v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha -v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha -v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha -v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha -v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha -v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha -v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha -v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v16, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v17, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v28, v4 -v_mov_b32 v29, v5 -v_mov_b32 v30, v6 -v_mov_b32 v31, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan -v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan -v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] -v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan -v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan -v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] -v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword -buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v34, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v35, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v36, v4 -v_mov_b32 v37, v5 -v_mov_b32 v38, v6 -v_mov_b32 v39, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan -v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan -v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] -v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan -v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan -v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] -v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword -buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v42, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v43, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v44, v4 -v_mov_b32 v45, v5 -v_mov_b32 v46, v6 -v_mov_b32 v47, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan -v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan -v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan -v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] -v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword -buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) -v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v52, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_and_b32 v4, v53, v10 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias -v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v56, v4 -v_mov_b32 v57, v5 -v_mov_b32 v58, v6 -v_mov_b32 v59, v7 -v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan -v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan -v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] -v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword -v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan -v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] -v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 -v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan -v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] -v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword -buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End_2 // jump to end -label_GW_B1_E1_M: -s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 -s_cbranch_scc1 label_To_Activation_Abs_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 -s_cbranch_scc1 label_To_Activation_Clippedrelu_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 -s_cbranch_scc1 label_To_Activation_Gelu_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 -s_cbranch_scc1 label_To_Activation_Leakyrelu_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 -s_cbranch_scc1 label_To_Activation_Relu_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 -s_cbranch_scc1 label_To_Activation_Sigmoid_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 -s_cbranch_scc1 label_To_Activation_Tanh_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 -s_cbranch_scc1 label_To_Activation_Geluscaling_VW1_beta_1_edge_1 // Branch if true -s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 -s_cbranch_scc1 label_To_Activation_Silu_VW1_beta_1_edge_1 // Branch if true -label_To_Activation_None_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_None_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Abs_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Abs_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Clippedrelu_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Clippedrelu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Gelu_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Gelu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Leakyrelu_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Leakyrelu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Relu_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Relu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Sigmoid_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Sigmoid_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Tanh_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Tanh_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Geluscaling_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Geluscaling_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_To_Activation_Silu_VW1_beta_1_edge_1: -s_getpc_b64 s[12:13] // addr of next instr -s_add_i32 s8, label_Activation_Silu_VW1, 0x4 // target branch offset -s_add_u32 s12, s12, s8 // add target branch offset -s_addc_u32 s13, s13, 0 // add high and carry -s_branch label_ActivationSetPCAddrEnd_1 -label_ActivationSetPCAddrEnd_1: - -/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #0 (d1,d0,vc1,vc0) = */ -/* (0,0,0,0:vw1); (0,0,0,1:vw1); (0,0,0,2:vw1); (0,0,0,3:vw1); (0,0,1,0:vw1); (0,0,1,1:vw1); (0,0,1,2:vw1); (0,0,1,3:vw1); (0,0,2,0:vw1); (0,0,2,1:vw1); (0,0,2,2:vw1); (0,0,2,3:vw1); (0,0,3,0:vw1); (0,0,3,1:vw1); (0,0,3,2:vw1); (0,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(0,0,0,0) */ -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -s_waitcnt lgkmcnt(0) // Wait for Bias LDS write -s_barrier // Bias LDS write barrier -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(0,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc0 // copy acc to vreg[0] -v_accvgpr_read_b32 v[vgprValuC+26], acc4 // copy acc to vreg[1] -v_accvgpr_read_b32 v[vgprValuC+33], acc8 // copy acc to vreg[2] -v_accvgpr_read_b32 v[vgprValuC+40], acc12 // copy acc to vreg[3] -v_accvgpr_read_b32 v[vgprValuC+45], acc1 // copy acc to vreg[4] -v_accvgpr_read_b32 v[vgprValuC+50], acc5 // copy acc to vreg[5] -v_accvgpr_read_b32 v[vgprValuC+55], acc9 // copy acc to vreg[6] -v_accvgpr_read_b32 v[vgprValuC+60], acc13 // copy acc to vreg[7] -v_accvgpr_read_b32 v[vgprValuC+65], acc2 // copy acc to vreg[8] -v_accvgpr_read_b32 v[vgprValuC+70], acc6 // copy acc to vreg[9] -v_accvgpr_read_b32 v[vgprValuC+75], acc10 // copy acc to vreg[10] -v_accvgpr_read_b32 v[vgprValuC+80], acc14 // copy acc to vreg[11] -v_accvgpr_read_b32 v[vgprValuC+85], acc3 // copy acc to vreg[12] -v_accvgpr_read_b32 v[vgprValuC+90], acc7 // copy acc to vreg[13] -v_accvgpr_read_b32 v[vgprValuC+95], acc11 // copy acc to vreg[14] -v_accvgpr_read_b32 v[vgprValuC+100], acc15 // copy acc to vreg[15] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #1 (d1,d0,vc1,vc0) = */ -/* (1,0,0,0:vw1); (1,0,0,1:vw1); (1,0,0,2:vw1); (1,0,0,3:vw1); (1,0,1,0:vw1); (1,0,1,1:vw1); (1,0,1,2:vw1); (1,0,1,3:vw1); (1,0,2,0:vw1); (1,0,2,1:vw1); (1,0,2,2:vw1); (1,0,2,3:vw1); (1,0,3,0:vw1); (1,0,3,1:vw1); (1,0,3,2:vw1); (1,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(1,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(1,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc16 // copy acc to vreg[16] -v_accvgpr_read_b32 v[vgprValuC+26], acc20 // copy acc to vreg[17] -v_accvgpr_read_b32 v[vgprValuC+33], acc24 // copy acc to vreg[18] -v_accvgpr_read_b32 v[vgprValuC+40], acc28 // copy acc to vreg[19] -v_accvgpr_read_b32 v[vgprValuC+45], acc17 // copy acc to vreg[20] -v_accvgpr_read_b32 v[vgprValuC+50], acc21 // copy acc to vreg[21] -v_accvgpr_read_b32 v[vgprValuC+55], acc25 // copy acc to vreg[22] -v_accvgpr_read_b32 v[vgprValuC+60], acc29 // copy acc to vreg[23] -v_accvgpr_read_b32 v[vgprValuC+65], acc18 // copy acc to vreg[24] -v_accvgpr_read_b32 v[vgprValuC+70], acc22 // copy acc to vreg[25] -v_accvgpr_read_b32 v[vgprValuC+75], acc26 // copy acc to vreg[26] -v_accvgpr_read_b32 v[vgprValuC+80], acc30 // copy acc to vreg[27] -v_accvgpr_read_b32 v[vgprValuC+85], acc19 // copy acc to vreg[28] -v_accvgpr_read_b32 v[vgprValuC+90], acc23 // copy acc to vreg[29] -v_accvgpr_read_b32 v[vgprValuC+95], acc27 // copy acc to vreg[30] -v_accvgpr_read_b32 v[vgprValuC+100], acc31 // copy acc to vreg[31] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #2 (d1,d0,vc1,vc0) = */ -/* (2,0,0,0:vw1); (2,0,0,1:vw1); (2,0,0,2:vw1); (2,0,0,3:vw1); (2,0,1,0:vw1); (2,0,1,1:vw1); (2,0,1,2:vw1); (2,0,1,3:vw1); (2,0,2,0:vw1); (2,0,2,1:vw1); (2,0,2,2:vw1); (2,0,2,3:vw1); (2,0,3,0:vw1); (2,0,3,1:vw1); (2,0,3,2:vw1); (2,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(2,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(2,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc32 // copy acc to vreg[32] -v_accvgpr_read_b32 v[vgprValuC+26], acc36 // copy acc to vreg[33] -v_accvgpr_read_b32 v[vgprValuC+33], acc40 // copy acc to vreg[34] -v_accvgpr_read_b32 v[vgprValuC+40], acc44 // copy acc to vreg[35] -v_accvgpr_read_b32 v[vgprValuC+45], acc33 // copy acc to vreg[36] -v_accvgpr_read_b32 v[vgprValuC+50], acc37 // copy acc to vreg[37] -v_accvgpr_read_b32 v[vgprValuC+55], acc41 // copy acc to vreg[38] -v_accvgpr_read_b32 v[vgprValuC+60], acc45 // copy acc to vreg[39] -v_accvgpr_read_b32 v[vgprValuC+65], acc34 // copy acc to vreg[40] -v_accvgpr_read_b32 v[vgprValuC+70], acc38 // copy acc to vreg[41] -v_accvgpr_read_b32 v[vgprValuC+75], acc42 // copy acc to vreg[42] -v_accvgpr_read_b32 v[vgprValuC+80], acc46 // copy acc to vreg[43] -v_accvgpr_read_b32 v[vgprValuC+85], acc35 // copy acc to vreg[44] -v_accvgpr_read_b32 v[vgprValuC+90], acc39 // copy acc to vreg[45] -v_accvgpr_read_b32 v[vgprValuC+95], acc43 // copy acc to vreg[46] -v_accvgpr_read_b32 v[vgprValuC+100], acc47 // copy acc to vreg[47] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #3 (d1,d0,vc1,vc0) = */ -/* (3,0,0,0:vw1); (3,0,0,1:vw1); (3,0,0,2:vw1); (3,0,0,3:vw1); (3,0,1,0:vw1); (3,0,1,1:vw1); (3,0,1,2:vw1); (3,0,1,3:vw1); (3,0,2,0:vw1); (3,0,2,1:vw1); (3,0,2,2:vw1); (3,0,2,3:vw1); (3,0,3,0:vw1); (3,0,3,1:vw1); (3,0,3,2:vw1); (3,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(3,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(3,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc48 // copy acc to vreg[48] -v_accvgpr_read_b32 v[vgprValuC+26], acc52 // copy acc to vreg[49] -v_accvgpr_read_b32 v[vgprValuC+33], acc56 // copy acc to vreg[50] -v_accvgpr_read_b32 v[vgprValuC+40], acc60 // copy acc to vreg[51] -v_accvgpr_read_b32 v[vgprValuC+45], acc49 // copy acc to vreg[52] -v_accvgpr_read_b32 v[vgprValuC+50], acc53 // copy acc to vreg[53] -v_accvgpr_read_b32 v[vgprValuC+55], acc57 // copy acc to vreg[54] -v_accvgpr_read_b32 v[vgprValuC+60], acc61 // copy acc to vreg[55] -v_accvgpr_read_b32 v[vgprValuC+65], acc50 // copy acc to vreg[56] -v_accvgpr_read_b32 v[vgprValuC+70], acc54 // copy acc to vreg[57] -v_accvgpr_read_b32 v[vgprValuC+75], acc58 // copy acc to vreg[58] -v_accvgpr_read_b32 v[vgprValuC+80], acc62 // copy acc to vreg[59] -v_accvgpr_read_b32 v[vgprValuC+85], acc51 // copy acc to vreg[60] -v_accvgpr_read_b32 v[vgprValuC+90], acc55 // copy acc to vreg[61] -v_accvgpr_read_b32 v[vgprValuC+95], acc59 // copy acc to vreg[62] -v_accvgpr_read_b32 v[vgprValuC+100], acc63 // copy acc to vreg[63] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #4 (d1,d0,vc1,vc0) = */ -/* (4,0,0,0:vw1); (4,0,0,1:vw1); (4,0,0,2:vw1); (4,0,0,3:vw1); (4,0,1,0:vw1); (4,0,1,1:vw1); (4,0,1,2:vw1); (4,0,1,3:vw1); (4,0,2,0:vw1); (4,0,2,1:vw1); (4,0,2,2:vw1); (4,0,2,3:vw1); (4,0,3,0:vw1); (4,0,3,1:vw1); (4,0,3,2:vw1); (4,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(4,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(4,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc64 // copy acc to vreg[64] -v_accvgpr_read_b32 v[vgprValuC+26], acc68 // copy acc to vreg[65] -v_accvgpr_read_b32 v[vgprValuC+33], acc72 // copy acc to vreg[66] -v_accvgpr_read_b32 v[vgprValuC+40], acc76 // copy acc to vreg[67] -v_accvgpr_read_b32 v[vgprValuC+45], acc65 // copy acc to vreg[68] -v_accvgpr_read_b32 v[vgprValuC+50], acc69 // copy acc to vreg[69] -v_accvgpr_read_b32 v[vgprValuC+55], acc73 // copy acc to vreg[70] -v_accvgpr_read_b32 v[vgprValuC+60], acc77 // copy acc to vreg[71] -v_accvgpr_read_b32 v[vgprValuC+65], acc66 // copy acc to vreg[72] -v_accvgpr_read_b32 v[vgprValuC+70], acc70 // copy acc to vreg[73] -v_accvgpr_read_b32 v[vgprValuC+75], acc74 // copy acc to vreg[74] -v_accvgpr_read_b32 v[vgprValuC+80], acc78 // copy acc to vreg[75] -v_accvgpr_read_b32 v[vgprValuC+85], acc67 // copy acc to vreg[76] -v_accvgpr_read_b32 v[vgprValuC+90], acc71 // copy acc to vreg[77] -v_accvgpr_read_b32 v[vgprValuC+95], acc75 // copy acc to vreg[78] -v_accvgpr_read_b32 v[vgprValuC+100], acc79 // copy acc to vreg[79] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 0, 1), (4, 0, 0, 2), (4, 0, 0, 3), (4, 0, 1, 0), (4, 0, 1, 1), (4, 0, 1, 2), (4, 0, 1, 3), (4, 0, 2, 0), (4, 0, 2, 1), (4, 0, 2, 2), (4, 0, 2, 3), (4, 0, 3, 0), (4, 0, 3, 1), (4, 0, 3, 2), (4, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #5 (d1,d0,vc1,vc0) = */ -/* (5,0,0,0:vw1); (5,0,0,1:vw1); (5,0,0,2:vw1); (5,0,0,3:vw1); (5,0,1,0:vw1); (5,0,1,1:vw1); (5,0,1,2:vw1); (5,0,1,3:vw1); (5,0,2,0:vw1); (5,0,2,1:vw1); (5,0,2,2:vw1); (5,0,2,3:vw1); (5,0,3,0:vw1); (5,0,3,1:vw1); (5,0,3,2:vw1); (5,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(5,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(5,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc80 // copy acc to vreg[80] -v_accvgpr_read_b32 v[vgprValuC+26], acc84 // copy acc to vreg[81] -v_accvgpr_read_b32 v[vgprValuC+33], acc88 // copy acc to vreg[82] -v_accvgpr_read_b32 v[vgprValuC+40], acc92 // copy acc to vreg[83] -v_accvgpr_read_b32 v[vgprValuC+45], acc81 // copy acc to vreg[84] -v_accvgpr_read_b32 v[vgprValuC+50], acc85 // copy acc to vreg[85] -v_accvgpr_read_b32 v[vgprValuC+55], acc89 // copy acc to vreg[86] -v_accvgpr_read_b32 v[vgprValuC+60], acc93 // copy acc to vreg[87] -v_accvgpr_read_b32 v[vgprValuC+65], acc82 // copy acc to vreg[88] -v_accvgpr_read_b32 v[vgprValuC+70], acc86 // copy acc to vreg[89] -v_accvgpr_read_b32 v[vgprValuC+75], acc90 // copy acc to vreg[90] -v_accvgpr_read_b32 v[vgprValuC+80], acc94 // copy acc to vreg[91] -v_accvgpr_read_b32 v[vgprValuC+85], acc83 // copy acc to vreg[92] -v_accvgpr_read_b32 v[vgprValuC+90], acc87 // copy acc to vreg[93] -v_accvgpr_read_b32 v[vgprValuC+95], acc91 // copy acc to vreg[94] -v_accvgpr_read_b32 v[vgprValuC+100], acc95 // copy acc to vreg[95] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(5, 0, 0, 0), (5, 0, 0, 1), (5, 0, 0, 2), (5, 0, 0, 3), (5, 0, 1, 0), (5, 0, 1, 1), (5, 0, 1, 2), (5, 0, 1, 3), (5, 0, 2, 0), (5, 0, 2, 1), (5, 0, 2, 2), (5, 0, 2, 3), (5, 0, 3, 0), (5, 0, 3, 1), (5, 0, 3, 2), (5, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #6 (d1,d0,vc1,vc0) = */ -/* (6,0,0,0:vw1); (6,0,0,1:vw1); (6,0,0,2:vw1); (6,0,0,3:vw1); (6,0,1,0:vw1); (6,0,1,1:vw1); (6,0,1,2:vw1); (6,0,1,3:vw1); (6,0,2,0:vw1); (6,0,2,1:vw1); (6,0,2,2:vw1); (6,0,2,3:vw1); (6,0,3,0:vw1); (6,0,3,1:vw1); (6,0,3,2:vw1); (6,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(6,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(6,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc96 // copy acc to vreg[96] -v_accvgpr_read_b32 v[vgprValuC+26], acc100 // copy acc to vreg[97] -v_accvgpr_read_b32 v[vgprValuC+33], acc104 // copy acc to vreg[98] -v_accvgpr_read_b32 v[vgprValuC+40], acc108 // copy acc to vreg[99] -v_accvgpr_read_b32 v[vgprValuC+45], acc97 // copy acc to vreg[100] -v_accvgpr_read_b32 v[vgprValuC+50], acc101 // copy acc to vreg[101] -v_accvgpr_read_b32 v[vgprValuC+55], acc105 // copy acc to vreg[102] -v_accvgpr_read_b32 v[vgprValuC+60], acc109 // copy acc to vreg[103] -v_accvgpr_read_b32 v[vgprValuC+65], acc98 // copy acc to vreg[104] -v_accvgpr_read_b32 v[vgprValuC+70], acc102 // copy acc to vreg[105] -v_accvgpr_read_b32 v[vgprValuC+75], acc106 // copy acc to vreg[106] -v_accvgpr_read_b32 v[vgprValuC+80], acc110 // copy acc to vreg[107] -v_accvgpr_read_b32 v[vgprValuC+85], acc99 // copy acc to vreg[108] -v_accvgpr_read_b32 v[vgprValuC+90], acc103 // copy acc to vreg[109] -v_accvgpr_read_b32 v[vgprValuC+95], acc107 // copy acc to vreg[110] -v_accvgpr_read_b32 v[vgprValuC+100], acc111 // copy acc to vreg[111] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 0, 1), (6, 0, 0, 2), (6, 0, 0, 3), (6, 0, 1, 0), (6, 0, 1, 1), (6, 0, 1, 2), (6, 0, 1, 3), (6, 0, 2, 0), (6, 0, 2, 1), (6, 0, 2, 2), (6, 0, 2, 3), (6, 0, 3, 0), (6, 0, 3, 1), (6, 0, 3, 2), (6, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #7 (d1,d0,vc1,vc0) = */ -/* (7,0,0,0:vw1); (7,0,0,1:vw1); (7,0,0,2:vw1); (7,0,0,3:vw1); (7,0,1,0:vw1); (7,0,1,1:vw1); (7,0,1,2:vw1); (7,0,1,3:vw1); (7,0,2,0:vw1); (7,0,2,1:vw1); (7,0,2,2:vw1); (7,0,2,3:vw1); (7,0,3,0:vw1); (7,0,3,1:vw1); (7,0,3,2:vw1); (7,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(7,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(7,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc112 // copy acc to vreg[112] -v_accvgpr_read_b32 v[vgprValuC+26], acc116 // copy acc to vreg[113] -v_accvgpr_read_b32 v[vgprValuC+33], acc120 // copy acc to vreg[114] -v_accvgpr_read_b32 v[vgprValuC+40], acc124 // copy acc to vreg[115] -v_accvgpr_read_b32 v[vgprValuC+45], acc113 // copy acc to vreg[116] -v_accvgpr_read_b32 v[vgprValuC+50], acc117 // copy acc to vreg[117] -v_accvgpr_read_b32 v[vgprValuC+55], acc121 // copy acc to vreg[118] -v_accvgpr_read_b32 v[vgprValuC+60], acc125 // copy acc to vreg[119] -v_accvgpr_read_b32 v[vgprValuC+65], acc114 // copy acc to vreg[120] -v_accvgpr_read_b32 v[vgprValuC+70], acc118 // copy acc to vreg[121] -v_accvgpr_read_b32 v[vgprValuC+75], acc122 // copy acc to vreg[122] -v_accvgpr_read_b32 v[vgprValuC+80], acc126 // copy acc to vreg[123] -v_accvgpr_read_b32 v[vgprValuC+85], acc115 // copy acc to vreg[124] -v_accvgpr_read_b32 v[vgprValuC+90], acc119 // copy acc to vreg[125] -v_accvgpr_read_b32 v[vgprValuC+95], acc123 // copy acc to vreg[126] -v_accvgpr_read_b32 v[vgprValuC+100], acc127 // copy acc to vreg[127] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 0, 1), (7, 0, 0, 2), (7, 0, 0, 3), (7, 0, 1, 0), (7, 0, 1, 1), (7, 0, 1, 2), (7, 0, 1, 3), (7, 0, 2, 0), (7, 0, 2, 1), (7, 0, 2, 2), (7, 0, 2, 3), (7, 0, 3, 0), (7, 0, 3, 1), (7, 0, 3, 2), (7, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #8 (d1,d0,vc1,vc0) = */ -/* (8,0,0,0:vw1); (8,0,0,1:vw1); (8,0,0,2:vw1); (8,0,0,3:vw1); (8,0,1,0:vw1); (8,0,1,1:vw1); (8,0,1,2:vw1); (8,0,1,3:vw1); (8,0,2,0:vw1); (8,0,2,1:vw1); (8,0,2,2:vw1); (8,0,2,3:vw1); (8,0,3,0:vw1); (8,0,3,1:vw1); (8,0,3,2:vw1); (8,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(8,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(8,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc128 // copy acc to vreg[128] -v_accvgpr_read_b32 v[vgprValuC+26], acc132 // copy acc to vreg[129] -v_accvgpr_read_b32 v[vgprValuC+33], acc136 // copy acc to vreg[130] -v_accvgpr_read_b32 v[vgprValuC+40], acc140 // copy acc to vreg[131] -v_accvgpr_read_b32 v[vgprValuC+45], acc129 // copy acc to vreg[132] -v_accvgpr_read_b32 v[vgprValuC+50], acc133 // copy acc to vreg[133] -v_accvgpr_read_b32 v[vgprValuC+55], acc137 // copy acc to vreg[134] -v_accvgpr_read_b32 v[vgprValuC+60], acc141 // copy acc to vreg[135] -v_accvgpr_read_b32 v[vgprValuC+65], acc130 // copy acc to vreg[136] -v_accvgpr_read_b32 v[vgprValuC+70], acc134 // copy acc to vreg[137] -v_accvgpr_read_b32 v[vgprValuC+75], acc138 // copy acc to vreg[138] -v_accvgpr_read_b32 v[vgprValuC+80], acc142 // copy acc to vreg[139] -v_accvgpr_read_b32 v[vgprValuC+85], acc131 // copy acc to vreg[140] -v_accvgpr_read_b32 v[vgprValuC+90], acc135 // copy acc to vreg[141] -v_accvgpr_read_b32 v[vgprValuC+95], acc139 // copy acc to vreg[142] -v_accvgpr_read_b32 v[vgprValuC+100], acc143 // copy acc to vreg[143] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 0, 1), (8, 0, 0, 2), (8, 0, 0, 3), (8, 0, 1, 0), (8, 0, 1, 1), (8, 0, 1, 2), (8, 0, 1, 3), (8, 0, 2, 0), (8, 0, 2, 1), (8, 0, 2, 2), (8, 0, 2, 3), (8, 0, 3, 0), (8, 0, 3, 1), (8, 0, 3, 2), (8, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #9 (d1,d0,vc1,vc0) = */ -/* (9,0,0,0:vw1); (9,0,0,1:vw1); (9,0,0,2:vw1); (9,0,0,3:vw1); (9,0,1,0:vw1); (9,0,1,1:vw1); (9,0,1,2:vw1); (9,0,1,3:vw1); (9,0,2,0:vw1); (9,0,2,1:vw1); (9,0,2,2:vw1); (9,0,2,3:vw1); (9,0,3,0:vw1); (9,0,3,1:vw1); (9,0,3,2:vw1); (9,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(9,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(9,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc144 // copy acc to vreg[144] -v_accvgpr_read_b32 v[vgprValuC+26], acc148 // copy acc to vreg[145] -v_accvgpr_read_b32 v[vgprValuC+33], acc152 // copy acc to vreg[146] -v_accvgpr_read_b32 v[vgprValuC+40], acc156 // copy acc to vreg[147] -v_accvgpr_read_b32 v[vgprValuC+45], acc145 // copy acc to vreg[148] -v_accvgpr_read_b32 v[vgprValuC+50], acc149 // copy acc to vreg[149] -v_accvgpr_read_b32 v[vgprValuC+55], acc153 // copy acc to vreg[150] -v_accvgpr_read_b32 v[vgprValuC+60], acc157 // copy acc to vreg[151] -v_accvgpr_read_b32 v[vgprValuC+65], acc146 // copy acc to vreg[152] -v_accvgpr_read_b32 v[vgprValuC+70], acc150 // copy acc to vreg[153] -v_accvgpr_read_b32 v[vgprValuC+75], acc154 // copy acc to vreg[154] -v_accvgpr_read_b32 v[vgprValuC+80], acc158 // copy acc to vreg[155] -v_accvgpr_read_b32 v[vgprValuC+85], acc147 // copy acc to vreg[156] -v_accvgpr_read_b32 v[vgprValuC+90], acc151 // copy acc to vreg[157] -v_accvgpr_read_b32 v[vgprValuC+95], acc155 // copy acc to vreg[158] -v_accvgpr_read_b32 v[vgprValuC+100], acc159 // copy acc to vreg[159] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 0, 1), (9, 0, 0, 2), (9, 0, 0, 3), (9, 0, 1, 0), (9, 0, 1, 1), (9, 0, 1, 2), (9, 0, 1, 3), (9, 0, 2, 0), (9, 0, 2, 1), (9, 0, 2, 2), (9, 0, 2, 3), (9, 0, 3, 0), (9, 0, 3, 1), (9, 0, 3, 2), (9, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #10 (d1,d0,vc1,vc0) = */ -/* (10,0,0,0:vw1); (10,0,0,1:vw1); (10,0,0,2:vw1); (10,0,0,3:vw1); (10,0,1,0:vw1); (10,0,1,1:vw1); (10,0,1,2:vw1); (10,0,1,3:vw1); (10,0,2,0:vw1); (10,0,2,1:vw1); (10,0,2,2:vw1); (10,0,2,3:vw1); (10,0,3,0:vw1); (10,0,3,1:vw1); (10,0,3,2:vw1); (10,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(10,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(10,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc160 // copy acc to vreg[160] -v_accvgpr_read_b32 v[vgprValuC+26], acc164 // copy acc to vreg[161] -v_accvgpr_read_b32 v[vgprValuC+33], acc168 // copy acc to vreg[162] -v_accvgpr_read_b32 v[vgprValuC+40], acc172 // copy acc to vreg[163] -v_accvgpr_read_b32 v[vgprValuC+45], acc161 // copy acc to vreg[164] -v_accvgpr_read_b32 v[vgprValuC+50], acc165 // copy acc to vreg[165] -v_accvgpr_read_b32 v[vgprValuC+55], acc169 // copy acc to vreg[166] -v_accvgpr_read_b32 v[vgprValuC+60], acc173 // copy acc to vreg[167] -v_accvgpr_read_b32 v[vgprValuC+65], acc162 // copy acc to vreg[168] -v_accvgpr_read_b32 v[vgprValuC+70], acc166 // copy acc to vreg[169] -v_accvgpr_read_b32 v[vgprValuC+75], acc170 // copy acc to vreg[170] -v_accvgpr_read_b32 v[vgprValuC+80], acc174 // copy acc to vreg[171] -v_accvgpr_read_b32 v[vgprValuC+85], acc163 // copy acc to vreg[172] -v_accvgpr_read_b32 v[vgprValuC+90], acc167 // copy acc to vreg[173] -v_accvgpr_read_b32 v[vgprValuC+95], acc171 // copy acc to vreg[174] -v_accvgpr_read_b32 v[vgprValuC+100], acc175 // copy acc to vreg[175] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(10, 0, 0, 0), (10, 0, 0, 1), (10, 0, 0, 2), (10, 0, 0, 3), (10, 0, 1, 0), (10, 0, 1, 1), (10, 0, 1, 2), (10, 0, 1, 3), (10, 0, 2, 0), (10, 0, 2, 1), (10, 0, 2, 2), (10, 0, 2, 3), (10, 0, 3, 0), (10, 0, 3, 1), (10, 0, 3, 2), (10, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #11 (d1,d0,vc1,vc0) = */ -/* (11,0,0,0:vw1); (11,0,0,1:vw1); (11,0,0,2:vw1); (11,0,0,3:vw1); (11,0,1,0:vw1); (11,0,1,1:vw1); (11,0,1,2:vw1); (11,0,1,3:vw1); (11,0,2,0:vw1); (11,0,2,1:vw1); (11,0,2,2:vw1); (11,0,2,3:vw1); (11,0,3,0:vw1); (11,0,3,1:vw1); (11,0,3,2:vw1); (11,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(11,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(11,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc176 // copy acc to vreg[176] -v_accvgpr_read_b32 v[vgprValuC+26], acc180 // copy acc to vreg[177] -v_accvgpr_read_b32 v[vgprValuC+33], acc184 // copy acc to vreg[178] -v_accvgpr_read_b32 v[vgprValuC+40], acc188 // copy acc to vreg[179] -v_accvgpr_read_b32 v[vgprValuC+45], acc177 // copy acc to vreg[180] -v_accvgpr_read_b32 v[vgprValuC+50], acc181 // copy acc to vreg[181] -v_accvgpr_read_b32 v[vgprValuC+55], acc185 // copy acc to vreg[182] -v_accvgpr_read_b32 v[vgprValuC+60], acc189 // copy acc to vreg[183] -v_accvgpr_read_b32 v[vgprValuC+65], acc178 // copy acc to vreg[184] -v_accvgpr_read_b32 v[vgprValuC+70], acc182 // copy acc to vreg[185] -v_accvgpr_read_b32 v[vgprValuC+75], acc186 // copy acc to vreg[186] -v_accvgpr_read_b32 v[vgprValuC+80], acc190 // copy acc to vreg[187] -v_accvgpr_read_b32 v[vgprValuC+85], acc179 // copy acc to vreg[188] -v_accvgpr_read_b32 v[vgprValuC+90], acc183 // copy acc to vreg[189] -v_accvgpr_read_b32 v[vgprValuC+95], acc187 // copy acc to vreg[190] -v_accvgpr_read_b32 v[vgprValuC+100], acc191 // copy acc to vreg[191] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(11, 0, 0, 0), (11, 0, 0, 1), (11, 0, 0, 2), (11, 0, 0, 3), (11, 0, 1, 0), (11, 0, 1, 1), (11, 0, 1, 2), (11, 0, 1, 3), (11, 0, 2, 0), (11, 0, 2, 1), (11, 0, 2, 2), (11, 0, 2, 3), (11, 0, 3, 0), (11, 0, 3, 1), (11, 0, 3, 2), (11, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #12 (d1,d0,vc1,vc0) = */ -/* (12,0,0,0:vw1); (12,0,0,1:vw1); (12,0,0,2:vw1); (12,0,0,3:vw1); (12,0,1,0:vw1); (12,0,1,1:vw1); (12,0,1,2:vw1); (12,0,1,3:vw1); (12,0,2,0:vw1); (12,0,2,1:vw1); (12,0,2,2:vw1); (12,0,2,3:vw1); (12,0,3,0:vw1); (12,0,3,1:vw1); (12,0,3,2:vw1); (12,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(12,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(12,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc192 // copy acc to vreg[192] -v_accvgpr_read_b32 v[vgprValuC+26], acc196 // copy acc to vreg[193] -v_accvgpr_read_b32 v[vgprValuC+33], acc200 // copy acc to vreg[194] -v_accvgpr_read_b32 v[vgprValuC+40], acc204 // copy acc to vreg[195] -v_accvgpr_read_b32 v[vgprValuC+45], acc193 // copy acc to vreg[196] -v_accvgpr_read_b32 v[vgprValuC+50], acc197 // copy acc to vreg[197] -v_accvgpr_read_b32 v[vgprValuC+55], acc201 // copy acc to vreg[198] -v_accvgpr_read_b32 v[vgprValuC+60], acc205 // copy acc to vreg[199] -v_accvgpr_read_b32 v[vgprValuC+65], acc194 // copy acc to vreg[200] -v_accvgpr_read_b32 v[vgprValuC+70], acc198 // copy acc to vreg[201] -v_accvgpr_read_b32 v[vgprValuC+75], acc202 // copy acc to vreg[202] -v_accvgpr_read_b32 v[vgprValuC+80], acc206 // copy acc to vreg[203] -v_accvgpr_read_b32 v[vgprValuC+85], acc195 // copy acc to vreg[204] -v_accvgpr_read_b32 v[vgprValuC+90], acc199 // copy acc to vreg[205] -v_accvgpr_read_b32 v[vgprValuC+95], acc203 // copy acc to vreg[206] -v_accvgpr_read_b32 v[vgprValuC+100], acc207 // copy acc to vreg[207] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 0, 1), (12, 0, 0, 2), (12, 0, 0, 3), (12, 0, 1, 0), (12, 0, 1, 1), (12, 0, 1, 2), (12, 0, 1, 3), (12, 0, 2, 0), (12, 0, 2, 1), (12, 0, 2, 2), (12, 0, 2, 3), (12, 0, 3, 0), (12, 0, 3, 1), (12, 0, 3, 2), (12, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #13 (d1,d0,vc1,vc0) = */ -/* (13,0,0,0:vw1); (13,0,0,1:vw1); (13,0,0,2:vw1); (13,0,0,3:vw1); (13,0,1,0:vw1); (13,0,1,1:vw1); (13,0,1,2:vw1); (13,0,1,3:vw1); (13,0,2,0:vw1); (13,0,2,1:vw1); (13,0,2,2:vw1); (13,0,2,3:vw1); (13,0,3,0:vw1); (13,0,3,1:vw1); (13,0,3,2:vw1); (13,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(13,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(13,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc208 // copy acc to vreg[208] -v_accvgpr_read_b32 v[vgprValuC+26], acc212 // copy acc to vreg[209] -v_accvgpr_read_b32 v[vgprValuC+33], acc216 // copy acc to vreg[210] -v_accvgpr_read_b32 v[vgprValuC+40], acc220 // copy acc to vreg[211] -v_accvgpr_read_b32 v[vgprValuC+45], acc209 // copy acc to vreg[212] -v_accvgpr_read_b32 v[vgprValuC+50], acc213 // copy acc to vreg[213] -v_accvgpr_read_b32 v[vgprValuC+55], acc217 // copy acc to vreg[214] -v_accvgpr_read_b32 v[vgprValuC+60], acc221 // copy acc to vreg[215] -v_accvgpr_read_b32 v[vgprValuC+65], acc210 // copy acc to vreg[216] -v_accvgpr_read_b32 v[vgprValuC+70], acc214 // copy acc to vreg[217] -v_accvgpr_read_b32 v[vgprValuC+75], acc218 // copy acc to vreg[218] -v_accvgpr_read_b32 v[vgprValuC+80], acc222 // copy acc to vreg[219] -v_accvgpr_read_b32 v[vgprValuC+85], acc211 // copy acc to vreg[220] -v_accvgpr_read_b32 v[vgprValuC+90], acc215 // copy acc to vreg[221] -v_accvgpr_read_b32 v[vgprValuC+95], acc219 // copy acc to vreg[222] -v_accvgpr_read_b32 v[vgprValuC+100], acc223 // copy acc to vreg[223] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(13, 0, 0, 0), (13, 0, 0, 1), (13, 0, 0, 2), (13, 0, 0, 3), (13, 0, 1, 0), (13, 0, 1, 1), (13, 0, 1, 2), (13, 0, 1, 3), (13, 0, 2, 0), (13, 0, 2, 1), (13, 0, 2, 2), (13, 0, 2, 3), (13, 0, 3, 0), (13, 0, 3, 1), (13, 0, 3, 2), (13, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #14 (d1,d0,vc1,vc0) = */ -/* (14,0,0,0:vw1); (14,0,0,1:vw1); (14,0,0,2:vw1); (14,0,0,3:vw1); (14,0,1,0:vw1); (14,0,1,1:vw1); (14,0,1,2:vw1); (14,0,1,3:vw1); (14,0,2,0:vw1); (14,0,2,1:vw1); (14,0,2,2:vw1); (14,0,2,3:vw1); (14,0,3,0:vw1); (14,0,3,1:vw1); (14,0,3,2:vw1); (14,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(14,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(14,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc224 // copy acc to vreg[224] -v_accvgpr_read_b32 v[vgprValuC+26], acc228 // copy acc to vreg[225] -v_accvgpr_read_b32 v[vgprValuC+33], acc232 // copy acc to vreg[226] -v_accvgpr_read_b32 v[vgprValuC+40], acc236 // copy acc to vreg[227] -v_accvgpr_read_b32 v[vgprValuC+45], acc225 // copy acc to vreg[228] -v_accvgpr_read_b32 v[vgprValuC+50], acc229 // copy acc to vreg[229] -v_accvgpr_read_b32 v[vgprValuC+55], acc233 // copy acc to vreg[230] -v_accvgpr_read_b32 v[vgprValuC+60], acc237 // copy acc to vreg[231] -v_accvgpr_read_b32 v[vgprValuC+65], acc226 // copy acc to vreg[232] -v_accvgpr_read_b32 v[vgprValuC+70], acc230 // copy acc to vreg[233] -v_accvgpr_read_b32 v[vgprValuC+75], acc234 // copy acc to vreg[234] -v_accvgpr_read_b32 v[vgprValuC+80], acc238 // copy acc to vreg[235] -v_accvgpr_read_b32 v[vgprValuC+85], acc227 // copy acc to vreg[236] -v_accvgpr_read_b32 v[vgprValuC+90], acc231 // copy acc to vreg[237] -v_accvgpr_read_b32 v[vgprValuC+95], acc235 // copy acc to vreg[238] -v_accvgpr_read_b32 v[vgprValuC+100], acc239 // copy acc to vreg[239] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 0, 1), (14, 0, 0, 2), (14, 0, 0, 3), (14, 0, 1, 0), (14, 0, 1, 1), (14, 0, 1, 2), (14, 0, 1, 3), (14, 0, 2, 0), (14, 0, 2, 1), (14, 0, 2, 2), (14, 0, 2, 3), (14, 0, 3, 0), (14, 0, 3, 1), (14, 0, 3, 2), (14, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ - -/******************************************/ -/* Global Write Beta Edge Batch #15 (d1,d0,vc1,vc0) = */ -/* (15,0,0,0:vw1); (15,0,0,1:vw1); (15,0,0,2:vw1); (15,0,0,3:vw1); (15,0,1,0:vw1); (15,0,1,1:vw1); (15,0,1,2:vw1); (15,0,1,3:vw1); (15,0,2,0:vw1); (15,0,2,1:vw1); (15,0,2,2:vw1); (15,0,2,3:vw1); (15,0,3,0:vw1); (15,0,3,1:vw1); (15,0,3,2:vw1); (15,0,3,3:vw1) */ -/******************************************/ - -/* calc coords, apply mask, and issue loads (if necessary) */ -v_mov_b32 v101, BufferOOB -/* (d1,vc1,d0,vc0)=(15,0,0,0) */ -v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride -v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row -s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride -v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v14, v0, s60 -v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE -v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v17, v14 offset:0 // load bias -v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v21, v4, s60 -v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE -v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v24, v21 offset:0 // load bias -v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v28, v4, s60 -v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE -v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v31, v28 offset:0 // load bias -v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,0,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v35, v4, s60 -v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE -v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset -ds_read_b32 v38, v35 offset:0 // load bias -v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE -buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI -v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v42, v0, s60 -v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE -v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v47, v4, s60 -v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE -v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v52, v4, s60 -v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE -v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,1,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v57, v4, s60 -v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE -v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v62, v0, s60 -v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE -v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v67, v4, s60 -v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE -v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v72, v4, s60 -v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE -v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,2,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v77, v4, s60 -v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE -v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,0) */ -v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 - -/* Fix for UseInitialStridesCD, emitAddressSetupCode */ -v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row -v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row -v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v82, v0, s60 -v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE -v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,1) */ -v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v87, v4, s60 -v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE -v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,2) */ -v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v92, v4, s60 -v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE -v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset -/* (d1,vc1,d0,vc0)=(15,3,0,3) */ -v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 -v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 -v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 -s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 -v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset -buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C -s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 -v_sub_u32 v97, v4, s60 -v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE -v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset -v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE -v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr -v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset -v_accvgpr_read_b32 v[vgprValuC+19], acc240 // copy acc to vreg[240] -v_accvgpr_read_b32 v[vgprValuC+26], acc244 // copy acc to vreg[241] -v_accvgpr_read_b32 v[vgprValuC+33], acc248 // copy acc to vreg[242] -v_accvgpr_read_b32 v[vgprValuC+40], acc252 // copy acc to vreg[243] -v_accvgpr_read_b32 v[vgprValuC+45], acc241 // copy acc to vreg[244] -v_accvgpr_read_b32 v[vgprValuC+50], acc245 // copy acc to vreg[245] -v_accvgpr_read_b32 v[vgprValuC+55], acc249 // copy acc to vreg[246] -v_accvgpr_read_b32 v[vgprValuC+60], acc253 // copy acc to vreg[247] -v_accvgpr_read_b32 v[vgprValuC+65], acc242 // copy acc to vreg[248] -v_accvgpr_read_b32 v[vgprValuC+70], acc246 // copy acc to vreg[249] -v_accvgpr_read_b32 v[vgprValuC+75], acc250 // copy acc to vreg[250] -v_accvgpr_read_b32 v[vgprValuC+80], acc254 // copy acc to vreg[251] -v_accvgpr_read_b32 v[vgprValuC+85], acc243 // copy acc to vreg[252] -v_accvgpr_read_b32 v[vgprValuC+90], acc247 // copy acc to vreg[253] -v_accvgpr_read_b32 v[vgprValuC+95], acc251 // copy acc to vreg[254] -v_accvgpr_read_b32 v[vgprValuC+100], acc255 // copy acc to vreg[255] -s_nop 1 // 2 wait states required before reading vgpr - -/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 0, 1), (15, 0, 0, 2), (15, 0, 0, 3), (15, 0, 1, 0), (15, 0, 1, 1), (15, 0, 1, 2), (15, 0, 1, 3), (15, 0, 2, 0), (15, 0, 2, 1), (15, 0, 2, 2), (15, 0, 2, 3), (15, 0, 3, 0), (15, 0, 3, 1), (15, 0, 3, 2), (15, 0, 3, 3)] */ -v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha -v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha -v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha -v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha -v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha -v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha -v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha -v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha -v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha -v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha -v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha -v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha -v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha -v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha -v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha -v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha -s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS - -/* apply mask, calc new C and issue writes */ -v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit -v_mov_b32 v11, 0x7fff0000 // fp32 Nan -v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+19] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v19, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan -v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] -v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 -buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+26] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v26, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan -v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] -v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 -buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+33] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v33, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan -v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] -v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 -buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+40] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v40, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan -v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] -v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 -buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+45] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v45, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan -v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] -v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 -buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+50] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v50, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan -v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] -v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 -buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+55] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v55, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan -v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] -v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 -buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+60] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v60, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan -v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] -v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 -buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+65] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v65, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan -v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] -v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 -buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+70] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v70, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan -v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] -v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 -buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+75] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v75, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan -v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] -v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 -buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+80] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v80, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan -v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] -v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 -buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v17, v[vgprValuC+85] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v85, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan -v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] -v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 -buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v24, v[vgprValuC+90] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v90, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan -v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] -v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 -buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v31, v[vgprValuC+95] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v95, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan -v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] -v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 -buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? -v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 -v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul -v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. -v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta -v_add_f32 v4, v38, v[vgprValuC+100] // C += bias -s_swappc_b64 s[58:59], s[12:13] -v_mov_b32 v100, v4 -v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan -v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 -v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding -v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] -v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 -buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D -s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst -s_branch label_GW_End_2 // jump to end -label_Activation_None_VW1: -s_setpc_b64 s[58:59] -label_Activation_Abs_VW1: -v_and_b32 v4, 0x7fffffff, v4 // Remove sign bit -s_setpc_b64 s[58:59] -label_Activation_Clippedrelu_VW1: -v_cmp_gt_f32 vcc, v4, s[sgpractivationAlpha] // x > alpha ? -v_min_f32 v4, s[sgpractivationBeta], v4 // min(x, beta) -v_cndmask_b32 v4, 0.0, v4, vcc // set x to 0 if <= alpha -s_setpc_b64 s[58:59] -label_Activation_Gelu_VW1: -v_mul_f32 v8, 0x3d372713, v4 // k1 * x -v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) -v_mul_f32 v4, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -s_setpc_b64 s[58:59] -label_Activation_Leakyrelu_VW1: -v_mul_f32 v8, s[sgpractivationAlpha], v4 // tmp = x * alpha -v_cmp_ge_f32 vcc, v4, 0.0 // x >= 0 ? -v_cndmask_b32 v4, v8, v4, vcc // set x to tmp if < 0 -s_setpc_b64 s[58:59] -label_Activation_Relu_VW1: -v_max_f32 v4, v4, 0 // x = max(0, x) -s_setpc_b64 s[58:59] -label_Activation_Sigmoid_VW1: -v_mul_f32 v4, 0xbfb8aa3b, v4 // (fused -1.442695) -v_exp_f32 v4, v4 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v4, 1.0, v4 // 1 + exp(-x) -v_rcp_f32 v4, v4 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -s_setpc_b64 s[58:59] -label_Activation_Tanh_VW1: -v_mul_f32 v4, s[sgpractivationAlpha], v4 // x * alpha -v_mul_f32 v4, 0x4038aa3b, v4 // (fused 2) -v_exp_f32 v4, v4 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v4, 1.0, v4 // e^2x + 1 -v_rcp_f32 v4, v4 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v4, -2.0, v4, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 -v_mul_f32 v4, s[sgpractivationBeta], v4 // beta * tanh(x) -s_setpc_b64 s[58:59] -label_Activation_Geluscaling_VW1: -v_mul_f32 v8, 0x3d372713, v4 // k1 * x -v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) -v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v4, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale -s_setpc_b64 s[58:59] -label_Activation_Silu_VW1: -v_mul_f32 v8, -1.4426950408889634, v4 // (fused -1.442695) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // 1 + exp(-x) -v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v4, v4, v8 // x / (1 + exp(-x)) -s_setpc_b64 s[58:59] -label_GW_End_2: -label_KernelEnd: -s_endpgm // Kernel End -label_Activation_None_VW4: -s_setpc_b64 s[58:59] -label_Activation_Abs_VW4: -v_and_b32 v4, 0x7fffffff, v4 // Remove sign bit -v_and_b32 v5, 0x7fffffff, v5 // Remove sign bit -v_and_b32 v6, 0x7fffffff, v6 // Remove sign bit -v_and_b32 v7, 0x7fffffff, v7 // Remove sign bit -s_setpc_b64 s[58:59] -label_Activation_Clippedrelu_VW4: -v_cmp_gt_f32 vcc, v4, s[sgpractivationAlpha] // x > alpha ? -v_min_f32 v4, s[sgpractivationBeta], v4 // min(x, beta) -v_cndmask_b32 v4, 0.0, v4, vcc // set x to 0 if <= alpha -v_cmp_gt_f32 vcc, v5, s[sgpractivationAlpha] // x > alpha ? -v_min_f32 v5, s[sgpractivationBeta], v5 // min(x, beta) -v_cndmask_b32 v5, 0.0, v5, vcc // set x to 0 if <= alpha -v_cmp_gt_f32 vcc, v6, s[sgpractivationAlpha] // x > alpha ? -v_min_f32 v6, s[sgpractivationBeta], v6 // min(x, beta) -v_cndmask_b32 v6, 0.0, v6, vcc // set x to 0 if <= alpha -v_cmp_gt_f32 vcc, v7, s[sgpractivationAlpha] // x > alpha ? -v_min_f32 v7, s[sgpractivationBeta], v7 // min(x, beta) -v_cndmask_b32 v7, 0.0, v7, vcc // set x to 0 if <= alpha -s_setpc_b64 s[58:59] -label_Activation_Gelu_VW4: -v_mul_f32 v8, 0x3d372713, v4 // k1 * x -v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) -v_mul_f32 v4, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v8, 0x3d372713, v5 // k1 * x -v_fma_f32 v8, v5, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v5, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v5, v8 // x * (1 + tanh(...)) -v_mul_f32 v5, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v8, 0x3d372713, v6 // k1 * x -v_fma_f32 v8, v6, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v6, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v6, v8 // x * (1 + tanh(...)) -v_mul_f32 v6, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v8, 0x3d372713, v7 // k1 * x -v_fma_f32 v8, v7, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v7, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v7, v8 // x * (1 + tanh(...)) -v_mul_f32 v7, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -s_setpc_b64 s[58:59] -label_Activation_Leakyrelu_VW4: -v_mul_f32 v8, s[sgpractivationAlpha], v4 // tmp = x * alpha -v_cmp_ge_f32 vcc, v4, 0.0 // x >= 0 ? -v_cndmask_b32 v4, v8, v4, vcc // set x to tmp if < 0 -v_mul_f32 v8, s[sgpractivationAlpha], v5 // tmp = x * alpha -v_cmp_ge_f32 vcc, v5, 0.0 // x >= 0 ? -v_cndmask_b32 v5, v8, v5, vcc // set x to tmp if < 0 -v_mul_f32 v8, s[sgpractivationAlpha], v6 // tmp = x * alpha -v_cmp_ge_f32 vcc, v6, 0.0 // x >= 0 ? -v_cndmask_b32 v6, v8, v6, vcc // set x to tmp if < 0 -v_mul_f32 v8, s[sgpractivationAlpha], v7 // tmp = x * alpha -v_cmp_ge_f32 vcc, v7, 0.0 // x >= 0 ? -v_cndmask_b32 v7, v8, v7, vcc // set x to tmp if < 0 -s_setpc_b64 s[58:59] -label_Activation_Relu_VW4: -v_max_f32 v4, v4, 0 // x = max(0, x) -v_max_f32 v5, v5, 0 // x = max(0, x) -v_max_f32 v6, v6, 0 // x = max(0, x) -v_max_f32 v7, v7, 0 // x = max(0, x) -s_setpc_b64 s[58:59] -label_Activation_Sigmoid_VW4: -v_mul_f32 v4, 0xbfb8aa3b, v4 // (fused -1.442695) -v_exp_f32 v4, v4 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v4, 1.0, v4 // 1 + exp(-x) -v_rcp_f32 v4, v4 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v5, 0xbfb8aa3b, v5 // (fused -1.442695) -v_exp_f32 v5, v5 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v5, 1.0, v5 // 1 + exp(-x) -v_rcp_f32 v5, v5 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v6, 0xbfb8aa3b, v6 // (fused -1.442695) -v_exp_f32 v6, v6 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v6, 1.0, v6 // 1 + exp(-x) -v_rcp_f32 v6, v6 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v7, 0xbfb8aa3b, v7 // (fused -1.442695) -v_exp_f32 v7, v7 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v7, 1.0, v7 // 1 + exp(-x) -v_rcp_f32 v7, v7 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -s_setpc_b64 s[58:59] -label_Activation_Tanh_VW4: -v_mul_f32 v4, s[sgpractivationAlpha], v4 // x * alpha -v_mul_f32 v4, 0x4038aa3b, v4 // (fused 2) -v_exp_f32 v4, v4 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v4, 1.0, v4 // e^2x + 1 -v_rcp_f32 v4, v4 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v4, -2.0, v4, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 -v_mul_f32 v4, s[sgpractivationBeta], v4 // beta * tanh(x) -v_mul_f32 v5, s[sgpractivationAlpha], v5 // x * alpha -v_mul_f32 v5, 0x4038aa3b, v5 // (fused 2) -v_exp_f32 v5, v5 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v5, 1.0, v5 // e^2x + 1 -v_rcp_f32 v5, v5 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v5, -2.0, v5, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 -v_mul_f32 v5, s[sgpractivationBeta], v5 // beta * tanh(x) -v_mul_f32 v6, s[sgpractivationAlpha], v6 // x * alpha -v_mul_f32 v6, 0x4038aa3b, v6 // (fused 2) -v_exp_f32 v6, v6 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v6, 1.0, v6 // e^2x + 1 -v_rcp_f32 v6, v6 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v6, -2.0, v6, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 -v_mul_f32 v6, s[sgpractivationBeta], v6 // beta * tanh(x) -v_mul_f32 v7, s[sgpractivationAlpha], v7 // x * alpha -v_mul_f32 v7, 0x4038aa3b, v7 // (fused 2) -v_exp_f32 v7, v7 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v7, 1.0, v7 // e^2x + 1 -v_rcp_f32 v7, v7 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v7, -2.0, v7, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 -v_mul_f32 v7, s[sgpractivationBeta], v7 // beta * tanh(x) -s_setpc_b64 s[58:59] -label_Activation_Geluscaling_VW4: -v_mul_f32 v8, 0x3d372713, v4 // k1 * x -v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) -v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v4, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale -v_mul_f32 v8, 0x3d372713, v5 // k1 * x -v_fma_f32 v8, v5, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v5, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v5, v8 // x * (1 + tanh(...)) -v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v5, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale -v_mul_f32 v8, 0x3d372713, v6 // k1 * x -v_fma_f32 v8, v6, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v6, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v6, v8 // x * (1 + tanh(...)) -v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v6, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale -v_mul_f32 v8, 0x3d372713, v7 // k1 * x -v_fma_f32 v8, v7, v8, 1.0 // 1 + (k1 * x * x) -v_mul_f32 v8, v7, v8 // x * (1 + k1 * x * x) -v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // e^2x + 1 -v_rcp_f32 v8, v8 // 1 / (e^2x + 1) -s_nop 0 // 1 wait states -v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) -v_mul_f32 v8, v7, v8 // x * (1 + tanh(...)) -v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) -v_mul_f32 v7, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale -s_setpc_b64 s[58:59] -label_Activation_Silu_VW4: -v_mul_f32 v8, -1.4426950408889634, v4 // (fused -1.442695) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // 1 + exp(-x) -v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v4, v4, v8 // x / (1 + exp(-x)) -v_mul_f32 v8, -1.4426950408889634, v5 // (fused -1.442695) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // 1 + exp(-x) -v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v5, v5, v8 // x / (1 + exp(-x)) -v_mul_f32 v8, -1.4426950408889634, v6 // (fused -1.442695) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // 1 + exp(-x) -v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v6, v6, v8 // x / (1 + exp(-x)) -v_mul_f32 v8, -1.4426950408889634, v7 // (fused -1.442695) -v_exp_f32 v8, v8 // exp step 2 -s_nop 0 // 1 wait states -v_add_f32 v8, 1.0, v8 // 1 + exp(-x) -v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) -s_nop 0 // 1 wait states -v_mul_f32 v7, v7, v8 // x / (1 + exp(-x)) -s_setpc_b64 s[58:59] -s_endpgm -label_ASM_End: /// The end of the kernel + +/******************************************/ +/* Begin Kernel */ +/******************************************/ +.amdgcn_target "amdgcn-amd-amdhsa--gfx942" +.text +.protected Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA +.globl Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA +.p2align 8 +.type Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA,@function +.section .rodata,#alloc +.p2align 6 +.amdhsa_kernel Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA + .amdhsa_user_sgpr_kernarg_segment_ptr 1 + .amdhsa_accum_offset 256 // accvgpr offset + .amdhsa_next_free_vgpr 512 // vgprs + .amdhsa_next_free_sgpr 84 // sgprs + .amdhsa_group_segment_fixed_size 40960 // lds bytes + .amdhsa_private_segment_fixed_size 0 + .amdhsa_system_sgpr_workgroup_id_x 1 + .amdhsa_system_sgpr_workgroup_id_y 1 + .amdhsa_system_sgpr_workgroup_id_z 1 + .amdhsa_system_vgpr_workitem_id 0 + .amdhsa_float_denorm_mode_32 3 + .amdhsa_float_denorm_mode_16_64 3 + .amdhsa_user_sgpr_count 13 + .amdhsa_user_sgpr_kernarg_preload_length 11 + .amdhsa_user_sgpr_kernarg_preload_offset 0 +.end_amdhsa_kernel +.text +/* Num VGPR =256 */ +/* Num AccVGPR=256 */ +/* Num SGPR =84 */ + +/******************************************/ +/* Optimizations and Config: */ +/******************************************/ +/* ThreadTile= 16 x 16 */ +/* SubGroup= 16 x 16 */ +/* VectorWidthA=4 */ +/* VectorWidthB=1 */ +/* GlobalReadVectorWidthA=8, GlobalReadVectorWidthB=8 */ +/* DirectToLdsA=False */ +/* DirectToLdsB=False */ +/* UseSgprForGRO=1 */ +.amdgpu_metadata +--- +custom.config: + ProblemType: + OperationType: GEMM + DataType: b + DestDataType: b + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: True + TransposeB: False + UseBias: 1 + Activation: True + UseScaleAlphaVec: 1 + UseBeta: True + Batched: True + GroupedGemm: False + SupportUserArgs: True + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: True + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] + 1LDSBuffer: 1 + ScheduleIterAlg: 3 + DepthU: 64 + GlobalReadVectorWidthA: 8 + GlobalReadVectorWidthB: 8 + AssertFree0ElementMultiple: 1 + AssertFree1ElementMultiple: 1 + AssertSummationElementMultiple: 1 + NoReject: True + InternalSupportParams: + KernArgsVersion: 0 + SupportUserGSU: True + SupportCustomWGM: True + SupportCustomStaggerU: True + UseUniversalArgs: True +amdhsa.version: + - 1 + - 1 +amdhsa.kernels: + - .name: Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA + .symbol: 'Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA.kd' + .language: OpenCL C + .language_version: + - 2 + - 0 + .args: + - .name: SizesFree0 + .size: 4 + .offset: 0 + .value_kind: by_value + .value_type: u32 + - .name: SizesFree1 + .size: 4 + .offset: 4 + .value_kind: by_value + .value_type: u32 + - .name: SizesFree2 + .size: 4 + .offset: 8 + .value_kind: by_value + .value_type: u32 + - .name: SizesSum0 + .size: 4 + .offset: 12 + .value_kind: by_value + .value_type: u32 + - .name: Gemm info + .size: 4 + .offset: 16 + .value_kind: by_value + .value_type: u32 + - .name: kernel info + .size: 4 + .offset: 20 + .value_kind: by_value + .value_type: u32 + - .name: D + .size: 8 + .offset: 24 + .value_kind: global_buffer + .value_type: bf16 + .address_space: generic + - .name: C + .size: 8 + .offset: 32 + .value_kind: global_buffer + .value_type: bf16 + .address_space: generic + - .name: A + .size: 8 + .offset: 40 + .value_kind: global_buffer + .value_type: bf16 + .address_space: generic + - .name: B + .size: 8 + .offset: 48 + .value_kind: global_buffer + .value_type: bf16 + .address_space: generic + - .name: strideD0 + .size: 4 + .offset: 56 + .value_kind: by_value + .value_type: u32 + - .name: strideD1 + .size: 4 + .offset: 60 + .value_kind: by_value + .value_type: u32 + - .name: strideC0 + .size: 4 + .offset: 64 + .value_kind: by_value + .value_type: u32 + - .name: strideC1 + .size: 4 + .offset: 68 + .value_kind: by_value + .value_type: u32 + - .name: strideA0 + .size: 4 + .offset: 72 + .value_kind: by_value + .value_type: u32 + - .name: strideA1 + .size: 4 + .offset: 76 + .value_kind: by_value + .value_type: u32 + - .name: strideB0 + .size: 4 + .offset: 80 + .value_kind: by_value + .value_type: u32 + - .name: strideB1 + .size: 4 + .offset: 84 + .value_kind: by_value + .value_type: u32 + - .name: alpha + .size: 4 + .offset: 88 + .value_kind: by_value + .value_type: f32 + - .name: beta + .size: 4 + .offset: 92 + .value_kind: by_value + .value_type: f32 + - .name: AddressScaleAlphaVec + .size: 8 + .offset: 96 + .value_kind: global_buffer + .value_type: f32 + .address_space: generic + - .name: bias + .size: 8 + .offset: 104 + .value_kind: global_buffer + .value_type: void + .address_space: generic + - .name: biasType + .size: 4 + .offset: 112 + .value_kind: by_value + .value_type: u32 + - .name: StrideBias + .size: 4 + .offset: 116 + .value_kind: by_value + .value_type: u32 + - .name: activationAlpha + .size: 4 + .offset: 120 + .value_kind: by_value + .value_type: f32 + - .name: activationBeta + .size: 4 + .offset: 124 + .value_kind: by_value + .value_type: f32 + - .name: activationType + .size: 4 + .offset: 128 + .value_kind: by_value + .value_type: u32 + .group_segment_fixed_size: 40960 + .kernarg_segment_align: 8 + .kernarg_segment_size: 136 + .max_flat_workgroup_size: 256 + .private_segment_fixed_size: 0 + .sgpr_count: 84 + .sgpr_spill_count: 0 + .vgpr_count: 256 + .vgpr_spill_count: 0 + .wavefront_size: 64 +... +.end_amdgpu_metadata +Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_DTVA: +label_ASM_Start: /// Main body of the asm kernel + +/* Magic div and mod functions */ +.macro V_MAGIC_DIV dstIdx:req dividend:req magicNumber:req magicShift:req magicA:req + v_mul_hi_u32 v[\dstIdx+1] \dividend \magicNumber + v_mul_lo_u32 v[\dstIdx+0] \dividend \magicA + v_add_u32 v[\dstIdx+0] v[\dstIdx+0] v[\dstIdx+1] + v_lshrrev_b32 v[\dstIdx+0] \magicShift v[\dstIdx+0] +.endm + +/******************************************/ +/* VGPR Assignments */ +/******************************************/ +/* ValuC range: [0-0), serializedStore enabled */ +.set vgprValuC, 0 +/* ValuA/B Xn=PLR buffer idx, In=InnerUnroll idx */ +.set vgprValuA_X0_I0_0, 0 +.set vgprValuA_X2_I0_0, 16 +.set vgprValuB_X0_I0, 32 +.set vgprValuB_X2_I0, 96 +.set vgprLocalWriteAddrA, 160 +.set vgprLocalWriteAddrB, 161 +.set vgprGlobalReadOffsetA, 162 +.set vgprGlobalReadOffsetB, 163 +.set vgprG2LB, 164 +.set vgprValuA_X0_I0_1, 196 +.set vgprValuA_X2_I0_1, 212 +.set vgprLocalReadAddrA, 228 +.set vgprLocalReadAddrB, 229 +.set vgprSerial, 230 + +/******************************************/ +/* SGPR Assignments */ +/******************************************/ +.set sgprKernArgAddress, 0 +.set sgprWorkGroup0, 2 +.set sgprWorkGroup1, 3 +.set sgprWorkGroup2, 4 +.set sgprArgType, 5 +.set sgprGSUSumIdx, 6 +.set sgprGSULog2BpeC, 8 +.set sgprGSULog2BpeD, 9 +.set sgprStaggerU, 10 +.set sgprWGM, 11 +.set sgprLoopCounterL, 12 +.set sgprOrigLoopCounter, 13 +.set sgprSrdD, 16 +.set sgprSrdC, 20 +.set sgprNumWorkGroups0, 14 +.set sgprNumWorkGroups1, 15 +.set sgprSizesFree, 24 +.set sgprSizesSum, 27 +.set sgprAddressD, 28 +.set sgprAddressC, 30 +.set sgprAddressA, 32 +.set sgprAddressB, 34 +.set sgprStridesD, 36 +.set sgprStridesC, 38 +.set sgprStridesA, 40 +.set sgprStridesB, 42 +.set sgprAlpha, 44 +.set sgprBeta, 45 +.set sgprGSU, 46 + +/* Size Assignments */ +.set sgprSizeI, sgprSizesFree+0 +.set sgprSizeJ, sgprSizesFree+1 +.set sgprSizeK, sgprSizesFree+2 +.set sgprSizeL, sgprSizesSum+0 + +/* Stride Assignments */ +.set constStrideD0I, 1 +.set sgprStrideD1J, sgprStridesD+0 +.set sgprStrideDK, sgprStridesD+1 +.set constStrideC0I, 1 +.set sgprStrideC1J, sgprStridesC+0 +.set sgprStrideCK, sgprStridesC+1 +.set constStrideAL, 1 +.set sgprStrideA0I, sgprStridesA+0 +.set sgprStrideAK, sgprStridesA+1 +.set constStrideBL, 1 +.set sgprStrideB1J, sgprStridesB+0 +.set sgprStrideBK, sgprStridesB+1 + +.set MT0, 256 +.set MT1, 256 +.set DepthU, 64 +.set BpeA, 2 +.set BpeALog2, 1 +.set BpeB, 2 +.set BpeBLog2, 1 +.set BpeAGR, 2 +.set BpeAGRLog2, 1 +.set BpeBGR, 2 +.set BpeBGRLog2, 1 +/* Number of elements to shift-left SRD */ +.set SrdShiftLeftA, 8 +.set SrdShiftLeftB, 8 +/* 2GB limit - set offsets to -1 to exceed this and clamp */ +.set BufferLimit, 0xffffffff +.set BufferOOB, 0x80000000 + +/******************************************/ +/* Bits 127:96 of SRD. */ +/* hex: 0x00020000 */ +/* dst_sel_x (3b): 0 */ +/* dst_sel_y (3b): 0 */ +/* dst_sel_z (3b): 0 */ +/* dst_sel_w (3b): 0 */ +/* num_format (3b): 0 */ +/* data_format (4b): 4 */ +/* user_vm_enable (1b): 0 */ +/* user_vm_mode (1b): 0 */ +/* index_stride (2b): 0 */ +/* add_tid_enable (1b): 0 */ +/* _unusedA (3b): 0 */ +/* nv (1b): 0 */ +/* _unusedB (2b): 0 */ +/* type (2b): 0 */ +/******************************************/ +.set Srd127_96, 0x00020000 + +/* Global Offset A */ +.macro GLOBAL_OFFSET_A vgprAddr:req vgprTmp:req + v_and_b32 v[\vgprTmp+0], 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) + v_and_b32 v[\vgprAddr+0], 15, v[\vgprTmp+0] // 1. M offset: mIdx = wtid % MI_M(16) + v_mul_lo_u32 v[\vgprAddr+0], s[sgprStrideA0I], v[\vgprAddr+0] // 1. M offset: mOffset = mIdx * mStride(k) + v_lshlrev_b32 v[\vgprAddr+0], 0x2, v[\vgprAddr+0] // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) + v_and_b32 v[\vgprTmp+0], 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) + v_lshrrev_b32 v[\vgprTmp+0], 4, v[\vgprTmp+0] // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) + v_lshlrev_b32 v[\vgprTmp+0], 0x3, v[\vgprTmp+0] // 5. K offset: lrKOffset = kIdx * mStride(8) + v_add_u32 v[\vgprAddr+0], v[\vgprTmp+0], v[\vgprAddr+0] // 6. offset in wave: lrOffset = bnOffset + lrKOffset + v_lshrrev_b32 v[\vgprTmp+0], 6, v[vgprSerial] // 7. wave offset in M dimen: wtid = tid / dividedForWaveId(64) + v_and_b32 v[\vgprTmp+0], 3, v[\vgprTmp+0] // 7. wave offset in M dimen: wtid0 = wtid % num1DWaves(4) + v_mul_lo_u32 v[\vgprTmp+0], s[sgprStrideA0I], v[\vgprTmp+0] // 7. wave offset in M dimen: wOffset = wtid0 * s[sgprStrideA0I](8192) + v_lshlrev_b32 v[\vgprTmp+0], 0x6, v[\vgprTmp+0] // 7. wave offset in M dimen: wOffset = wOffset * 16 * vw(4) + v_add_u32 v[\vgprAddr+0], v[\vgprTmp+0], v[\vgprAddr+0] // 7. final local read offset: flrOffset = lrOffset + WOffset + v_add_u32 v[\vgprAddr+0] 0x8 v[\vgprAddr+0] // add prepad for pointer shift + v_lshlrev_b32 v[\vgprAddr+0] 0x1 v[\vgprAddr+0] // offset *= bytes/element +.endm + +/* Global Offset B */ +.macro GLOBAL_OFFSET_B vgprAddr:req vgprOffsetL:req vgprOffset1J:req vgprTmp:req + v_mul_lo_u32 v[\vgprTmp+0] s[sgprStrideB1J] v[\vgprOffset1J] // mul d1 lower + v_add_co_u32 v[\vgprAddr+0] vcc v[\vgprOffsetL] v[\vgprTmp+0] // accumulate K lower + v_add_u32 v[\vgprAddr+0] 0x8 v[\vgprAddr+0] // add prepad for pointer shift + v_lshlrev_b32 v[\vgprAddr+0] 0x1 v[\vgprAddr+0] // offset *= bytes/element +.endm + +/* Dynamic Scalar Divide: vQuotient=vDividend/vDivisor; vRemainder=vDividend%vDivisor; */ +.macro DYNAMIC_VECTOR_DIVIDE vQuotient vRemainder vDividend vDivisor vTmp0 vTmp1 sTmp + v_cvt_f32_u32 v[\vQuotient] v[\vDivisor] + v_rcp_f32 v[\vQuotient] v[\vQuotient] + v_mul_f32 v[\vQuotient] 0x4f800000 v[\vQuotient] + v_cvt_u32_f32 v[\vQuotient] v[\vQuotient] + v_mul_lo_u32 v[\vRemainder] v[\vDivisor] v[\vQuotient] + v_mul_hi_u32 v[\vTmp0] v[\vDivisor] v[\vQuotient] + v_sub_co_u32 v[\vTmp1] vcc 0x0 v[\vRemainder] + v_cmp_ne_i32 s[\sTmp:\sTmp+1] 0x0 v[\vTmp0] + v_cndmask_b32 v[\vRemainder] v[\vTmp1] v[\vRemainder] s[\sTmp:\sTmp+1] + v_mul_hi_u32 v[\vRemainder] v[\vRemainder] v[\vQuotient] + v_sub_co_u32 v[\vTmp0] vcc v[\vQuotient] v[\vRemainder] + v_add_co_u32 v[\vQuotient] vcc v[\vQuotient] v[\vRemainder] + v_cndmask_b32 v[\vQuotient] v[\vQuotient] v[\vTmp0] s[\sTmp:\sTmp+1] + v_mul_hi_u32 v[\vQuotient] v[\vQuotient] v[\vDividend] + v_mul_lo_u32 v[\vRemainder] v[\vQuotient] v[\vDivisor] + v_sub_co_u32 v[\vTmp0] vcc v[\vDividend] v[\vRemainder] + v_cmp_ge_u32 s[\sTmp:\sTmp+1] v[\vDividend] v[\vRemainder] + v_add_co_u32 v[\vRemainder] vcc 0x1 v[\vQuotient] + v_add_co_u32 v[\vTmp1] vcc -1 v[\vQuotient] + v_cmp_le_u32 vcc v[\vDivisor] v[\vTmp0] + s_and_b64 vcc s[\sTmp:\sTmp+1] vcc + v_cndmask_b32 v[\vQuotient] v[\vQuotient] v[\vRemainder] vcc + v_cndmask_b32 v[\vQuotient] v[\vTmp1] v[\vQuotient] s[\sTmp:\sTmp+1] + v_cmp_ne_i32 vcc 0x0 v[\vDivisor] + v_cndmask_b32 v[\vQuotient] -1 v[\vQuotient] vcc // final result + v_mul_lo_u32 v[\vRemainder] v[\vQuotient] v[\vDivisor] + v_sub_co_u32 v[\vRemainder] vcc v[\vDividend] v[\vRemainder] // final result +.endm + +/******************************************/ +/* Allocate Resources */ +/******************************************/ + +/* Load num of Gemms */ +s_load_dword s47, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x0 + +/* Load GSU data */ +s_load_dword s[sgprGSU], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x4 +s_waitcnt lgkmcnt(0) +s_lshr_b32 s48, s47, 0x1e // Get arg type +s_and_b32 s47, 0x3fffffff, s47 // Get nums of gemm +s_cmp_eq_u32 s48, 0 // Is kernel args +s_cbranch_scc0 label_HBMArgs +s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], 0x8 // Shift common args +s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 + +/* Load Kernel Args */ +s_load_dwordx16 s[24:39], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x0 +s_load_dwordx4 s[40:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x40 +s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 +s_waitcnt lgkmcnt(0) +s_branch label_LoadArgsEnd +label_HBMArgs: + +/* Load address of kernel arguments */ +s_load_dwordx2 s[sgprKernArgAddress:sgprKernArgAddress+1], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x8 +s_waitcnt lgkmcnt(0) // wait for args to load +label_LoadArgsEnd: +s_branch label_common_kernel_entry + +/* pad 41 snops to satisfy 0x100 code size for Preload Backward Compatibility Prologue */ +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +s_nop 0 +label_Preload_Offset_Start: +s_and_b32 s47, 0x3fffffff, s2 // Get nums of gemm +s_lshr_b32 s48, s2, 0x1e // Get arg type +s_mov_b32 s[sgprGSU], s3 // Preload internal args +s_cmp_eq_u32 s48, 0 // Is kernel args +s_cbranch_scc0 label_Preload_HBMArgs +s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], 0x8 // Shift common args +s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 + +/* Load Kernel Args */ +s_load_dword s33, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x24 +s_load_dwordx2 s[34:35], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x28 +s_load_dwordx8 s[36:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x30 +s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 +s_mov_b32 s24, s4 // move preload data to correct sgpr +s_mov_b32 s25, s5 // move preload data to correct sgpr +s_mov_b32 s26, s6 // move preload data to correct sgpr +s_mov_b32 s27, s7 // move preload data to correct sgpr +s_mov_b32 s28, s8 // move preload data to correct sgpr +s_mov_b32 s29, s9 // move preload data to correct sgpr +s_mov_b32 s30, s10 // move preload data to correct sgpr +s_mov_b32 s31, s11 // move preload data to correct sgpr +s_mov_b32 s32, s12 // move preload data to correct sgpr +s_branch label_Preload_LoadArgsEnd +label_Preload_HBMArgs: +s_mov_b64 s[sgprKernArgAddress:sgprKernArgAddress+1], s[4:5] // Load address of kernel arguments +label_Preload_LoadArgsEnd: +label_common_kernel_entry: /// for both preload/non-preload common code +s_mov_b32 s[sgprWorkGroup0+0], s13 // restore workgroup id +s_mov_b32 s[sgprWorkGroup0+1], s14 // restore workgroup id +s_mov_b32 s[sgprWorkGroup0+2], s15 // restore workgroup id +s_and_b32 s[sgprWGM], s[sgprGSU], 0xff00 // Restore WGM +s_lshr_b32 s[sgprWGM], s[sgprWGM], 0x8 +s_and_b32 s[sgprStaggerU], s[sgprGSU], 0xffff0000 // Restore StaggerU related vars +s_lshr_b32 s[sgprStaggerU], s[sgprStaggerU], 0x10 +s_and_b32 s[sgprGSU], s[sgprGSU], 0xff // Restore GSU +s_mov_b32 s[sgprArgType], s48 +s_mov_b32 m0, 0xa000 // LDS clamp at 40960 bytes +v_mov_b32 v[vgprSerial], v0 // thread serial id +s_cmp_eq_u32 s48, 0 +s_cbranch_scc0 label_MultiGemm +/* init: add vgpr [0...160) to pool */ +/* init: add vgpr [0...0) to pool */ +/* init: add agpr [0...256) to pool */ + +/******************************************/ +/* Local Read Addresses */ +/******************************************/ + +/* local read addresses: tile assignments a/b */ +/* lr0I */ +v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v0, 15, v1 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v0, 0x6, v0 // 1. N offset: nOffset = nIdx * nStride(64) +/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ +v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) +v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v1, 0x3, v1 // 5. K offset: lrKOffset = kIdx * mStride(8) +v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in N dimen: wtid = tid / dividedForWaveId(64) +v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) +v_lshlrev_b32 v1, 0xc, v1 // 7. wave offset in M dimen: wOffset = wtid0 * W0Stride(4096) +v_add_u32 v0, v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset +/* lr1J */ +v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v1, 0x6, v1 // 1. N offset: nOffset = nIdx * nStride(64) +/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ + // 4. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) +v_and_b32 v2, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v2, 4, v2 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v2, 0x3, v2 // 5. K offset: lrKOffset = kIdx * mStride(8) +v_add_u32 v1, v2, v1 // 6. offset in wave: lrOffset = bnOffset + lrKOffset + +/* local read addresses: final offsets a */ +v_lshrrev_b32 v2, 6, v[vgprSerial] // v2 = v[vgprSerial] / 64 +v_lshrrev_b32 v2, 2, v2 // LSU offset: Get LSU wave_id +s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True +v_mul_lo_u32 v2, s49, v2 // LSU offset: lsuoffset = wave_id*lsuStride*(MT0+PAD) +v_add_lshl_u32 v[vgprLocalReadAddrA], v2, v0, 0x1 // Final Offset: offset = (lro0+lsuoffset)*bpeDS +v_lshrrev_b32 v3, 9, v[vgprLocalReadAddrA] // Final Offset: padding 32 per block 512 +v_lshlrev_b32 v3, 0x5, v3 // Final Offset: padding 32 per block 512 +v_add_u32 v[vgprLocalReadAddrA], v3, v[vgprLocalReadAddrA] // Final Offset: add padding 32 per block 512 + +/* local read addresses: final offsets b */ +v_lshrrev_b32 v0, 6, v[vgprSerial] // v0 = v[vgprSerial] / 64 +v_lshrrev_b32 v0, 2, v0 // LSU offset: Get LSU wave_id +s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True +v_mul_lo_u32 v0, s49, v0 // LSU offset: lsuoffset = wave_id*lsuStride*(MT1+PAD) +v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v1, 0x1 // Final Offset: offset = (lro1+lsuoffset)*bpeDS +v_lshrrev_b32 v2, 7, v[vgprLocalReadAddrB] // Final Offset: padding 32 per block 128 +v_lshlrev_b32 v2, 0x5, v2 // Final Offset: padding 32 per block 128 +v_add_u32 v[vgprLocalReadAddrB], v2, v[vgprLocalReadAddrB] // Final Offset: add padding 32 per block 128 + +/* local read addresses: declare addresses a */ +/* N/A */ + +/* local read addresses: declare addresses b */ + +/******************************************/ +/* Local Write Addresses */ +/******************************************/ +/* LVCA = 8 */ +/* v1 = A-unroll = serial%LVCA */ +v_lshrrev_b32 v0, 3, v[vgprSerial] // v0 = v[vgprSerial] / 8 +v_and_b32 v1, 7, v[vgprSerial] // v1 = v[vgprSerial] % 8 +/* unroll *= glvw */ +v_lshlrev_b32 v1, 0x3, v1 // v1 = v1 * 8 +v_mov_b32 v4, v1 // copy for GlobalSplitU +/* LVCB = 8 */ +/* v3 = B-unroll = serial%LVCB */ +v_lshrrev_b32 v2, 3, v[vgprSerial] // v2 = v[vgprSerial] / 8 +v_and_b32 v3, 7, v[vgprSerial] // v3 = v[vgprSerial] % 8 +/* unroll *= glvw */ +v_lshlrev_b32 v3, 0x3, v3 // v3 = v3 * 8 +v_mov_b32 v5, v3 // copy for GlobalSplitU +/* lwaUnrollAssignmentA = v4 */ +/* lwaUnrollAssignmentB = v5 */ + +/* local write addresses: first offset a */ +v_mul_u32_u24 v[vgprLocalWriteAddrA], 0x40, v0 // lwAL**(DepthU_Compute + PAD) +v_add_lshl_u32 v[vgprLocalWriteAddrA], v4, v[vgprLocalWriteAddrA], 0x1 // lwFOA = (lwAA + lwAL*(DepthU+PAD))*bpeDS +v_lshrrev_b32 v6, 9, v[vgprLocalWriteAddrA] // padding 32 per block 512 +v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 512 +v_add_u32 v[vgprLocalWriteAddrA], v6, v[vgprLocalWriteAddrA] // add padding 32 per block 512 + +/* local write addresses: first offset b */ +v_mul_u32_u24 v[vgprLocalWriteAddrB], 0x40, v2 // lwBL**(DepthU_Compute + PAD) +v_add_lshl_u32 v[vgprLocalWriteAddrB], v5, v[vgprLocalWriteAddrB], 0x1 // lwFOB = (lwBB + lwBL*(DepthU+PAD))*bpeDS +v_lshrrev_b32 v6, 7, v[vgprLocalWriteAddrB] // padding 32 per block 128 +v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 128 +v_add_u32 v[vgprLocalWriteAddrB], v6, v[vgprLocalWriteAddrB] // add padding 32 per block 128 +v_mov_b32 v8, MT0 // set MT0 into sgpr +v_mov_b32 v7, s[sgprSizesFree+0] // set Free0 size +v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) +v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) +v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) +v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) +v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) +v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) +v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) +v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) +v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil +v_mov_b32 v8, MT1 // set MT1 into sgpr +v_mov_b32 v7, s[sgprSizesFree+1] // set Free1 size +v_readfirstlane_b32 s[sgprNumWorkGroups0], v6 // set back to numWorkGroup0 +v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) +v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) +v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) +v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) +v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) +v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) +v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) +v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) +v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil +s_nop 0 // 1 wait states +v_readfirstlane_b32 s[sgprNumWorkGroups1], v6 // set back to numWorkGroup1 +s_waitcnt lgkmcnt(0) // wait for 44/0 bytes of kern args +s_branch label_MultiGemmEnd +label_MultiGemm: + +/* Check if custom structure pointer is null */ +s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? +s_cbranch_scc1 label_IsExternalValid // branch if ArgType == 2 +s_mov_b32 s15, 124 +s_mul_i32 s54, s47, 4 +s_mov_b64 s[48:49], s[sgprKernArgAddress:sgprKernArgAddress+1] +s_branch label_IsExternalValidEnd +label_IsExternalValid: +s_mov_b32 s15, 196 +s_mov_b32 s54, 0x0 +s_mov_b64 s[48:49], s[sgprKernArgAddress:sgprKernArgAddress+1] +label_IsExternalValidEnd: + +/* Grouped Gemm:: prefetch 1 arg load */ +s_mov_b32 s14, 1 +s_mov_b32 s55, 0 +s_load_dwordx4 s[24:27], s[48:49], s54 +s_cmpk_eq_u32 s47, 1 // if gemm_count is 1? +s_cbranch_scc1 label_wgTable_noLoadLoop + +/* Grouped Gemm:: accumulate numTiles for each gemm */ +/* Grouped Gemm:: loop start */ +label_Loop_GemmCount: +s_waitcnt lgkmcnt(0) +s_lshr_b32 s52, s24, 8 // s52 = s24 / 256 +s_and_b32 s50, 255, s24 // s50 = s24 % 256 +s_addc_u32 s52, s52, 0x0 +s_lshr_b32 s53, s25, 8 // s53 = s25 / 256 +s_and_b32 s50, 255, s25 // s50 = s25 % 256 +s_addc_u32 s53, s53, 0x0 +s_mul_i32 s52, s52, s53 +s_mul_i32 s52, s52, s26 +s_mul_i32 s52, s52, s[sgprGSU] +s_add_u32 s55, s55, s52 +s_cmp_lt_u32 s[sgprWorkGroup0], s55 +s_cbranch_scc1 label_FOUND +s_add_u32 s54, s54, s15 +s_load_dwordx4 s[24:27], s[48:49], s54 +s_add_u32 s14, s14, 1 +s_cmp_lt_u32 s14, s47 +s_cbranch_scc1 label_Loop_GemmCount + +/* Grouped Gemm:: noLoadLoop */ +label_wgTable_noLoadLoop: +s_waitcnt lgkmcnt(0) +s_lshr_b32 s52, s24, 8 // s52 = s24 / 256 +s_and_b32 s50, 255, s24 // s50 = s24 % 256 +s_addc_u32 s52, s52, 0x0 +s_lshr_b32 s53, s25, 8 // s53 = s25 / 256 +s_and_b32 s50, 255, s25 // s50 = s25 % 256 +s_addc_u32 s53, s53, 0x0 +s_mul_i32 s52, s52, s53 +s_mul_i32 s52, s52, s26 +s_mul_i32 s52, s52, s[sgprGSU] +s_add_u32 s55, s55, s52 + +/* Grouped Gemm:: gemmIndex found */ +label_FOUND: +s_sub_u32 s49, s14, 1 +s_sub_u32 s48, s55, s52 +s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s48 +/* Check if custom structure pointer is null */ +s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? +s_cbranch_scc1 label_LoadExternalStruct // branch if ArgType == 2 + +/* Grouped Gemm: offset argument address to gemm */ +/* Grouped Gemm: offset address from wg_table_start to args_start */ +s_lshl2_add_u32 s[sgprKernArgAddress], s47, s[sgprKernArgAddress] +s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 +/* Grouped Gemm: offset address from args_start to gemm_start */ +s_mul_i32 s49, s49, 124 +s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], s49 +s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 + +/* Load Kernel Args */ +s_load_dwordx16 s[28:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x10 +s_load_dwordx2 s[44:45], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 +s_branch label_LoadExternalStructEnd +label_LoadExternalStruct: +/* Grouped Gemm: offset address from args_start to gemm_start */ +s_mul_i32 s49, s49, 196 +s_add_u32 s[sgprKernArgAddress], s[sgprKernArgAddress], s49 +s_addc_u32 s[sgprKernArgAddress+1], s[sgprKernArgAddress+1], 0x0 +s_load_dwordx16 s[28:43], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x10 +s_load_dword s44, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x50 +// Read Beta +s_load_dword s45, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x60 +label_LoadExternalStructEnd: +/* init: add vgpr [0...160) to pool */ +/* init: add vgpr [0...0) to pool */ +/* init: add agpr [0...256) to pool */ + +/******************************************/ +/* Local Read Addresses */ +/******************************************/ + +/* local read addresses: tile assignments a/b */ +/* lr0I */ +v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v0, 15, v1 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v0, 0x6, v0 // 1. N offset: nOffset = nIdx * nStride(64) +/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ +v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) +v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v1, 0x3, v1 // 5. K offset: lrKOffset = kIdx * mStride(8) +v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in N dimen: wtid = tid / dividedForWaveId(64) +v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) +v_lshlrev_b32 v1, 0xc, v1 // 7. wave offset in M dimen: wOffset = wtid0 * W0Stride(4096) +v_add_u32 v0, v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset +/* lr1J */ +v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v1, 0x6, v1 // 1. N offset: nOffset = nIdx * nStride(64) +/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ + // 4. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) +v_and_b32 v2, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v2, 4, v2 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v2, 0x3, v2 // 5. K offset: lrKOffset = kIdx * mStride(8) +v_add_u32 v1, v2, v1 // 6. offset in wave: lrOffset = bnOffset + lrKOffset + +/* local read addresses: final offsets a */ +v_lshrrev_b32 v2, 6, v[vgprSerial] // v2 = v[vgprSerial] / 64 +v_lshrrev_b32 v2, 2, v2 // LSU offset: Get LSU wave_id +s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True +v_mul_lo_u32 v2, s49, v2 // LSU offset: lsuoffset = wave_id*lsuStride*(MT0+PAD) +v_add_lshl_u32 v[vgprLocalReadAddrA], v2, v0, 0x1 // Final Offset: offset = (lro0+lsuoffset)*bpeDS +v_lshrrev_b32 v3, 9, v[vgprLocalReadAddrA] // Final Offset: padding 32 per block 512 +v_lshlrev_b32 v3, 0x5, v3 // Final Offset: padding 32 per block 512 +v_add_u32 v[vgprLocalReadAddrA], v3, v[vgprLocalReadAddrA] // Final Offset: add padding 32 per block 512 + +/* local read addresses: final offsets b */ +v_lshrrev_b32 v0, 6, v[vgprSerial] // v0 = v[vgprSerial] / 64 +v_lshrrev_b32 v0, 2, v0 // LSU offset: Get LSU wave_id +s_mov_b32 s49, 64 // LSU offset: stride = lsuStride(64) when umlds==True +v_mul_lo_u32 v0, s49, v0 // LSU offset: lsuoffset = wave_id*lsuStride*(MT1+PAD) +v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v1, 0x1 // Final Offset: offset = (lro1+lsuoffset)*bpeDS +v_lshrrev_b32 v2, 7, v[vgprLocalReadAddrB] // Final Offset: padding 32 per block 128 +v_lshlrev_b32 v2, 0x5, v2 // Final Offset: padding 32 per block 128 +v_add_u32 v[vgprLocalReadAddrB], v2, v[vgprLocalReadAddrB] // Final Offset: add padding 32 per block 128 + +/* local read addresses: declare addresses a */ +/* N/A */ + +/* local read addresses: declare addresses b */ + +/******************************************/ +/* Local Write Addresses */ +/******************************************/ +/* LVCA = 8 */ +/* v1 = A-unroll = serial%LVCA */ +v_lshrrev_b32 v0, 3, v[vgprSerial] // v0 = v[vgprSerial] / 8 +v_and_b32 v1, 7, v[vgprSerial] // v1 = v[vgprSerial] % 8 +/* unroll *= glvw */ +v_lshlrev_b32 v1, 0x3, v1 // v1 = v1 * 8 +v_mov_b32 v4, v1 // copy for GlobalSplitU +/* LVCB = 8 */ +/* v3 = B-unroll = serial%LVCB */ +v_lshrrev_b32 v2, 3, v[vgprSerial] // v2 = v[vgprSerial] / 8 +v_and_b32 v3, 7, v[vgprSerial] // v3 = v[vgprSerial] % 8 +/* unroll *= glvw */ +v_lshlrev_b32 v3, 0x3, v3 // v3 = v3 * 8 +v_mov_b32 v5, v3 // copy for GlobalSplitU +/* lwaUnrollAssignmentA = v4 */ +/* lwaUnrollAssignmentB = v5 */ + +/* local write addresses: first offset a */ +v_mul_u32_u24 v[vgprLocalWriteAddrA], 0x40, v0 // lwAL**(DepthU_Compute + PAD) +v_add_lshl_u32 v[vgprLocalWriteAddrA], v4, v[vgprLocalWriteAddrA], 0x1 // lwFOA = (lwAA + lwAL*(DepthU+PAD))*bpeDS +v_lshrrev_b32 v6, 9, v[vgprLocalWriteAddrA] // padding 32 per block 512 +v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 512 +v_add_u32 v[vgprLocalWriteAddrA], v6, v[vgprLocalWriteAddrA] // add padding 32 per block 512 + +/* local write addresses: first offset b */ +v_mul_u32_u24 v[vgprLocalWriteAddrB], 0x40, v2 // lwBL**(DepthU_Compute + PAD) +v_add_lshl_u32 v[vgprLocalWriteAddrB], v5, v[vgprLocalWriteAddrB], 0x1 // lwFOB = (lwBB + lwBL*(DepthU+PAD))*bpeDS +v_lshrrev_b32 v6, 7, v[vgprLocalWriteAddrB] // padding 32 per block 128 +v_lshlrev_b32 v6, 0x5, v6 // padding 32 per block 128 +v_add_u32 v[vgprLocalWriteAddrB], v6, v[vgprLocalWriteAddrB] // add padding 32 per block 128 +v_mov_b32 v8, MT0 // set MT0 into sgpr +v_mov_b32 v7, s[sgprSizesFree+0] // set Free0 size +v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) +v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) +v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) +v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) +v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) +v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) +v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) +v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) +v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil +v_mov_b32 v8, MT1 // set MT1 into sgpr +v_mov_b32 v7, s[sgprSizesFree+1] // set Free1 size +v_readfirstlane_b32 s[sgprNumWorkGroups0], v6 // set back to numWorkGroup0 +v_cvt_f32_u32 v6, v8 // v6 = ceil(v7 / v8) +v_rcp_iflag_f32 v6, v6 // v6 = ceil(v7 / v8) +v_cvt_f32_u32 v9, v7 // v6 = ceil(v7 / v8) +v_mul_f32 v6, v6, v9 // v6 = ceil(v7 / v8) +v_cvt_u32_f32 v6, v6 // v6 = ceil(v7 / v8) +v_mul_u32_u24 v9, v6, v8 // v6 = ceil(v7 / v8) +v_sub_u32 v9, v7, v9 // v6 = ceil(v7 / v8) +v_cmp_ne_u32 vcc, v9, 0 // v6 = ceil(v7 / v8) +v_addc_co_u32 v6, vcc, v6, 0, vcc // ceil +s_nop 0 // 1 wait states +v_readfirstlane_b32 s[sgprNumWorkGroups1], v6 // set back to numWorkGroup1 +s_waitcnt lgkmcnt(0) // wait for 44/0 bytes of kern args + +/* Early stop if N(SizeFreeJ) == 0 */ +s_cmp_eq_u32 s[sgprSizeJ], 0x0 +s_cbranch_scc0 label_NoEarlyStop_N0 +label_EarlyStop_if_N_is_0: +s_endpgm +label_NoEarlyStop_N0: + +/* Grouped Gemm: remap wg from 1D(idxWG012) to 3D(wg2,wg1,wg0) */ +/* wg2 = idxWG012 * smallMagicNumber(1/(numWG0*numWG1)) */ +s_mul_i32 s48, s[sgprNumWorkGroups0], s[sgprNumWorkGroups1] +s_mul_i32 s48, s48, s[sgprGSU] +v_cvt_f32_u32 v6, s48 // s48 = s[sgprWorkGroup0] / s48 +v_rcp_iflag_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s48 +v_cvt_f32_u32 v7, s[sgprWorkGroup0] // s48 = s[sgprWorkGroup0] / s48 +v_mul_f32 v6, v6, v7 // s48 = s[sgprWorkGroup0] / s48 +v_cvt_u32_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s48 +v_mul_u32_u24 v7, v6, s48 // s48 = s[sgprWorkGroup0] / s48 +v_sub_u32 v7, s[sgprWorkGroup0], v7 // s48 = s[sgprWorkGroup0] / s48 +v_cmpx_eq_u32 exec, v7, s48 // s48 = s[sgprWorkGroup0] / s48 +v_add_u32 v6, 1, v6 // s48 = s[sgprWorkGroup0] / s48 +s_mov_b64 exec, -1 // s48 = s[sgprWorkGroup0] / s48 +v_readfirstlane_b32 s48, v6 +s_mov_b32 s[sgprWorkGroup2], s48 +/* idxWG01 = idxWG012 - wg2 * numWG0 * numWG1 */ +s_mul_i32 s48, s[sgprNumWorkGroups1], s[sgprNumWorkGroups0] +s_mul_i32 s48, s48, s[sgprWorkGroup2] +s_mul_i32 s48, s48, s[sgprGSU] +s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s48 +/* wg1 = idxWG01 * smallMagicNumber(1/numWG0) */ +v_cvt_f32_u32 v6, s[sgprNumWorkGroups0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_rcp_iflag_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_cvt_f32_u32 v7, s[sgprWorkGroup0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_mul_f32 v6, v6, v7 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_cvt_u32_f32 v6, v6 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_mul_u32_u24 v7, v6, s[sgprNumWorkGroups0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_sub_u32 v7, s[sgprWorkGroup0], v7 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_cmpx_eq_u32 exec, v7, s[sgprNumWorkGroups0] // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_add_u32 v6, 1, v6 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +s_mov_b64 exec, -1 // s48 = s[sgprWorkGroup0] / s[sgprNumWorkGroups0] +v_readfirstlane_b32 s48, v6 +s_mov_b32 s[sgprWorkGroup1], s48 +/* wg0 = idxWG01 - wg1 * numWG0 */ +s_mul_i32 s48, s[sgprWorkGroup1], s[sgprNumWorkGroups0] +s_sub_u32 s[sgprWorkGroup0], s[sgprWorkGroup0], s48 + +/* Early stop if wg exceed */ +s_cmp_ge_u32 s[sgprWorkGroup2], s[sgprSizesFree+2] +s_cbranch_scc0 label_NoEarlyStop_wgExceed +label_EarlyStop_if_wg_exceed: +s_endpgm +label_NoEarlyStop_wgExceed: + +label_MultiGemmEnd: +.set sgprSrdA, 48 +.set sgprSrdB, 52 +.set sgprShadowLimitA, 56 +.set sgprShadowLimitB, 58 +.set sgprStaggerUIter, 47 +.set sgprWrapUA, 60 +.set sgprWrapUB, 62 +.set sgprGlobalReadIncsA, 64 +.set sgprGlobalReadIncsB, 65 +.set sgprScalarGlobalReadOffsetA, 66 +.set sgprScalarGlobalReadOffsetB, 73 +s_sub_u32 s[sgprAddressA+0], s[sgprAddressA+0], 16 // pre-pad to make room for possible pointer shift +s_subb_u32 s[sgprAddressA+1], s[sgprAddressA+1], 0 // pre-pad to make room for possible pointer shift +s_sub_u32 s[sgprAddressB+0], s[sgprAddressB+0], 16 // pre-pad to make room for possible pointer shift +s_subb_u32 s[sgprAddressB+1], s[sgprAddressB+1], 0 // pre-pad to make room for possible pointer shift + +/* Short circuit condition if Alpha == 0, then sumDims=0 */ +v_cmp_eq_f32 vcc, s[sgprAlpha], 0.0 // s[Alpha] == 0.0f ? +s_cbranch_vccz label_AlphaNonZero // branch if s[Alpha] != 0 +s_mov_b32 s[sgprSizesSum+0], 0x0 // Set summation dim=0 if Alpha == 0 +label_AlphaNonZero: + +/******************************************/ +/* Begin setupNewTile */ +/******************************************/ + +/* global read addresses: work-group */ +/* graWorkGroup mapping */ +s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? +s_cbranch_scc1 label_GSU // branch if GSU == 1 +// GSU-not-WGMapRR :nwg1 = (size1J + MT1J - 1) / MT1J; +v_cvt_f32_u32 v6, s[sgprGSU] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_rcp_iflag_f32 v6, v6 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_cvt_f32_u32 v7, s[sgprWorkGroup1] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_mul_f32 v6, v6, v7 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_cvt_u32_f32 v6, v6 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_mul_u32_u24 v7, v6, s[sgprGSU] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_sub_u32 v7, s[sgprWorkGroup1], v7 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_cmpx_eq_u32 exec, v7, s[sgprGSU] // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_add_u32 v6, 1, v6 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_mov_b32 v7, 0 // s[sgprGSUSumIdx] = s[sgprWorkGroup1] % s[sgprGSU] +s_mov_b64 exec, -1 // s[sgprWorkGroup1] = s[sgprWorkGroup1] / s[sgprGSU] +v_readfirstlane_b32 s[sgprWorkGroup1], v6 +v_readfirstlane_b32 s[sgprGSUSumIdx], v7 +s_mov_b32 s[sgprGSULog2BpeC], 1 +s_mov_b32 s[sgprGSULog2BpeD], 2 +s_branch label_GSU_End +label_GSU: +s_mov_b64 s[sgprGSUSumIdx:sgprGSUSumIdx+1], 0 // Set GSUSumIdx to 0 +s_mov_b32 s[sgprGSULog2BpeC], 1 +s_mov_b32 s[sgprGSULog2BpeD], 1 +label_GSU_End: +s_cmp_le_u32 s[sgprWGM], 1 // WGM <= 1 ? +s_cbranch_scc1 label_WGM // branch if WGM <= 1 +v_cvt_f32_u32 v6, s[sgprWGM] // WGM +v_rcp_iflag_f32 v6, v6 // WGM +v_cvt_f32_u32 v7, s[sgprWorkGroup1] // WGM +v_mul_f32 v6, v6, v7 // WGM +v_cvt_u32_f32 v6, v6 // WGM +v_mul_u32_u24 v7, v6, s[sgprWGM] // WGM +v_sub_u32 v7, s[sgprWorkGroup1], v7 // WGM +v_cmpx_eq_u32 exec, v7, s[sgprWGM] // WGM +v_add_u32 v6, 1, v6 // WGM +s_mov_b64 exec, -1 // WGM +v_readfirstlane_b32 s82, v6 +s_mul_i32 s83, s82, s[sgprWGM] // quotient * non-magic divisor +s_sub_u32 s83, s[sgprWorkGroup1], s83 // WorkGroup1=remainder +s_mul_i32 s83, s83, s[sgprNumWorkGroups0] // (wg1 % WGM)*nwg0 +s_add_u32 s83, s83, s[sgprWorkGroup0] // wgSerial = wg0 + (wg1 % WGM)*nwg0 +v_cvt_f32_u32 v6, s[sgprWGM] // WGM +v_rcp_iflag_f32 v6, v6 // WGM +v_cvt_f32_u32 v7, s[sgprNumWorkGroups1] // WGM +v_mul_f32 v6, v6, v7 // WGM +v_cvt_u32_f32 v6, v6 // WGM +v_mul_u32_u24 v7, v6, s[sgprWGM] // WGM +v_sub_u32 v7, s[sgprNumWorkGroups1], v7 // WGM +v_cmpx_eq_u32 exec, v7, s[sgprWGM] // WGM +v_add_u32 v6, 1, v6 // WGM +s_mov_b64 exec, -1 // WGM +v_readfirstlane_b32 s80, v6 +s_mul_i32 s81, s[sgprWGM], s80 // quotient * non-magic divisor +s_sub_u32 s81, s[sgprNumWorkGroups1], s81 // WorkGroup1=remainder +s_cmp_eq_u32 s81, 0 // remainder == 0 ? +s_cmov_b32 s81, s[sgprWGM] // remainder = WGM if remainder == 0 +s_cmp_ge_u32 s82, s80 // blockId >= numFullBlocks ? +s_cselect_b32 s80, s81, s[sgprWGM] +v_cvt_f32_u32 v6, s80 // s[sgprWorkGroup0] = s83 / s80 +v_rcp_iflag_f32 v6, v6 // s[sgprWorkGroup0] = s83 / s80 +v_cvt_f32_u32 v7, s83 // s[sgprWorkGroup0] = s83 / s80 +v_mul_f32 v6, v6, v7 // s[sgprWorkGroup0] = s83 / s80 +v_cvt_u32_f32 v6, v6 // s[sgprWorkGroup0] = s83 / s80 +v_mul_u32_u24 v7, v6, s80 // s[sgprWorkGroup0] = s83 / s80 +v_sub_u32 v7, s83, v7 // s[sgprWorkGroup0] = s83 / s80 +v_cmpx_eq_u32 exec, v7, s80 // s[sgprWorkGroup0] = s83 / s80 +v_add_u32 v6, 1, v6 // s[sgprWorkGroup0] = s83 / s80 +v_mov_b32 v7, 0 // s[sgprWorkGroup1] = s83 % s80 +s_mov_b64 exec, -1 // s[sgprWorkGroup0] = s83 / s80 +v_readfirstlane_b32 s[sgprWorkGroup0], v6 +v_readfirstlane_b32 s[sgprWorkGroup1], v7 +s_mul_i32 s82, s82, s[sgprWGM] // blockId * WGM +s_add_u32 s[sgprWorkGroup1], s[sgprWorkGroup1], s82 // wg1 += blockId * WGM +label_WGM: + +/* global read addresses: tile offset assignment a */ +/* graTileAssignmentA = v0 */ + +/* global read addresses: tile offset assignment b */ +/* graTileAssignmentB = v2 */ + +/* global read addresses: unroll assignment a */ +/* v1 */ + +/* global read addresses: unroll assignment b */ +/* v3 */ + +/* global read addresses: other free assignments */ +/* s[sgprWorkGroup2] */ + +/* global read addresses: tile offsets a */ + +/* global read addresses: tile offsets b */ + +/* global read addresses: unroll offsets a */ + +/* global read addresses: unroll offsets b */ + +/* global read addresses: final offsets a */ +GLOBAL_OFFSET_A vgprGlobalReadOffsetA+0, 6 +s_mul_i32 s[sgprScalarGlobalReadOffsetA+0], s[sgprStrideA0I], 1 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+0], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetA+1], s[sgprStrideA0I], 2 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+1], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetA+2], s[sgprStrideA0I], 3 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+2], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetA+3], 1, 32 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+3], s[sgprScalarGlobalReadOffsetA+3], 0x1 // scalar offset *= bytes/element +s_add_u32 s[sgprScalarGlobalReadOffsetA+4], s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+3] +s_add_u32 s[sgprScalarGlobalReadOffsetA+5], s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+3] +s_add_u32 s[sgprScalarGlobalReadOffsetA+6], s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+3] + +/* global read addresses: final offsets b */ +GLOBAL_OFFSET_B vgprGlobalReadOffsetB+0, 3, 2, 6 // gROB_0_0_0_0 +s_mul_i32 s[sgprScalarGlobalReadOffsetB+0], s[sgprStrideB1J], 32 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+0], s[sgprScalarGlobalReadOffsetB+0], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetB+1], s[sgprStrideB1J], 64 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+1], s[sgprScalarGlobalReadOffsetB+1], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetB+2], s[sgprStrideB1J], 96 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+2], s[sgprScalarGlobalReadOffsetB+2], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetB+3], s[sgprStrideB1J], 128 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+3], s[sgprScalarGlobalReadOffsetB+3], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetB+4], s[sgprStrideB1J], 160 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+4], s[sgprScalarGlobalReadOffsetB+4], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetB+5], s[sgprStrideB1J], 192 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+5], s[sgprScalarGlobalReadOffsetB+5], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetB+6], s[sgprStrideB1J], 224 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetB+6], s[sgprScalarGlobalReadOffsetB+6], 0x1 // scalar offset *= bytes/element + +/* global read addresses: addresses a */ +/* max read offset = size[n] * stride[n-1] */ +s_mul_hi_u32 s83, s[sgprWorkGroup0], 256 // WorkGroup[01] * MT +s_mul_i32 s82, s[sgprWorkGroup0], 256 // WorkGroup[01] * MT +s_mul_hi_u32 s83, s82, s[sgprStrideA0I] // tlu=0, scaled tile-offset by stride +s_mul_i32 s82, s82, s[sgprStrideA0I] // tlu=0, scaled tile-offset by stride +s_mul_hi_u32 s81, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx +s_mul_i32 s80, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx +s_add_u32 s82, s82, s80 // accum GsuOffset term to tilestart +s_addc_u32 s83, s83, s81 // accum GsuOffset term to tilestart +s_mov_b32 s[sgprShadowLimitA+0], 1 // Init tensor size +s_mov_b32 s[sgprShadowLimitA+1], 0 // init tensor size +s_sub_u32 s80, s[sgprSizeL], 1 // (size-1) +s_mul_hi_u32 s81, constStrideAL, s80 // stride x (size-1) +s_mul_i32 s80, constStrideAL, s80 // stride x (size-1) +s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // sum tensor size +s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // sum tensor size +s_sub_u32 s80, s[sgprSizeI], 1 // (size-1) +s_mul_hi_u32 s81, s[sgprStrideA0I], s80 // stride x (size-1) +s_mul_i32 s80, s[sgprStrideA0I], s80 // stride x (size-1) +s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // sum tensor size +s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // sum tensor size +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s82 // sub tileStart +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s83 // sub tileStart +s_lshl_b64 s[sgprShadowLimitA:sgprShadowLimitA+1], s[sgprShadowLimitA:sgprShadowLimitA+1], 0x1 // Set limit to use bytes +s_add_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], 16 // extend limit for pre-pad +s_addc_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], 0 // extend limit for pre-pad +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +s_mul_hi_u32 s81, s[sgprStrideAK], s[sgprWorkGroup2] // Stride*WG +s_mul_i32 s80, s[sgprStrideAK], s[sgprWorkGroup2] // Stride*WG +s_add_u32 s82, s82, s80 // accum wg term to tilestart +s_addc_u32 s83, s83, s81 // accum wg term to tilestart +s_lshl_b64 s[82:83], s[82:83], 0x1 // tileStart *= BPE +s_add_u32 s[sgprSrdA+0], s[sgprAddressA+0], s82 // SRD base = Address+ tileStart0 +s_addc_u32 s[sgprSrdA+1], s[sgprAddressA+1], s83 // SRD base = Address+ tileStart1 +s_mov_b32 s[sgprSrdA+3], Srd127_96 // Set bits 127_96 in SRD + +/* global read addresses: addresses b */ +/* max read offset = size[n] * stride[n-1] */ +s_mul_hi_u32 s83, s[sgprWorkGroup1], 256 // WorkGroup[01] * MT +s_mul_i32 s82, s[sgprWorkGroup1], 256 // WorkGroup[01] * MT +s_mul_hi_u32 s83, s82, s[sgprStrideB1J] // tlu=0, scaled tile-offset by stride +s_mul_i32 s82, s82, s[sgprStrideB1J] // tlu=0, scaled tile-offset by stride +s_mul_hi_u32 s81, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx +s_mul_i32 s80, 64, s[sgprGSUSumIdx] // gsuOffset = DepthU*bpeGR*GSUSumIdx +s_add_u32 s82, s82, s80 // accum GsuOffset term to tilestart +s_addc_u32 s83, s83, s81 // accum GsuOffset term to tilestart +s_mov_b32 s[sgprShadowLimitB+0], 1 // Init tensor size +s_mov_b32 s[sgprShadowLimitB+1], 0 // init tensor size +s_sub_u32 s80, s[sgprSizeL], 1 // (size-1) +s_mul_hi_u32 s81, constStrideBL, s80 // stride x (size-1) +s_mul_i32 s80, constStrideBL, s80 // stride x (size-1) +s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // sum tensor size +s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // sum tensor size +s_sub_u32 s80, s[sgprSizeJ], 1 // (size-1) +s_mul_hi_u32 s81, s[sgprStrideB1J], s80 // stride x (size-1) +s_mul_i32 s80, s[sgprStrideB1J], s80 // stride x (size-1) +s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // sum tensor size +s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // sum tensor size +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s82 // sub tileStart +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s83 // sub tileStart +s_lshl_b64 s[sgprShadowLimitB:sgprShadowLimitB+1], s[sgprShadowLimitB:sgprShadowLimitB+1], 0x1 // Set limit to use bytes +s_add_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], 16 // extend limit for pre-pad +s_addc_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], 0 // extend limit for pre-pad +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +s_mul_hi_u32 s81, s[sgprStrideBK], s[sgprWorkGroup2] // Stride*WG +s_mul_i32 s80, s[sgprStrideBK], s[sgprWorkGroup2] // Stride*WG +s_add_u32 s82, s82, s80 // accum wg term to tilestart +s_addc_u32 s83, s83, s81 // accum wg term to tilestart +s_lshl_b64 s[82:83], s[82:83], 0x1 // tileStart *= BPE +s_add_u32 s[sgprSrdB+0], s[sgprAddressB+0], s82 // SRD base = Address+ tileStart0 +s_addc_u32 s[sgprSrdB+1], s[sgprAddressB+1], s83 // SRD base = Address+ tileStart1 +s_mov_b32 s[sgprSrdB+3], Srd127_96 // Set bits 127_96 in SRD +s_mul_i32 s80, s[sgprGSU], DepthU*BpeAGR +s_mov_b32 s[sgprGlobalReadIncsA+0], s80 // incrA (unrollIdx) + +/* global read addresses: increments b */ +s_mul_i32 s80, s[sgprGSU], DepthU*BpeBGR +s_mov_b32 s[sgprGlobalReadIncsB+0], s80 // incrB (unrollIdx) + +/* declare loop num iterations */ +s_lshr_b32 s[sgprLoopCounterL], s[sgprSizesSum+0], 6 // s[sgprLoopCounterL] = s[sgprSizesSum+0] / 64 +s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? +s_cbranch_scc1 label_GSU_1 // branch if GSU == 1 +v_cvt_f32_u32 v0, s[sgprGSU] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_rcp_iflag_f32 v0, v0 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_cvt_f32_u32 v1, s[sgprLoopCounterL] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_mul_f32 v0, v0, v1 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_cvt_u32_f32 v0, v0 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_mul_u32_u24 v1, v0, s[sgprGSU] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_sub_u32 v1, s[sgprLoopCounterL], v1 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_cmpx_eq_u32 exec, v1, s[sgprGSU] // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_add_u32 v0, 1, v0 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_mov_b32 v1, 0 // s[sgprGSUSumIdx+1] = s[sgprLoopCounterL] % s[sgprGSU] +s_mov_b64 exec, -1 // s[sgprLoopCounterL] = s[sgprLoopCounterL] / s[sgprGSU] +v_readfirstlane_b32 s[sgprLoopCounterL], v0 +v_readfirstlane_b32 s[sgprGSUSumIdx+1], v1 +s_add_u32 s80, 1, s[sgprLoopCounterL] // tmp<-numIterMyWg+ +s_cmp_lt_u32 s[sgprGSUSumIdx], s[sgprGSUSumIdx+1] // gsuSumIdx < numIterPerWgRemainder +s_cmov_b32 s[sgprLoopCounterL], s80 // numIterMyWg++ if needed +label_GSU_1: +s_mov_b32 s[sgprOrigLoopCounter], s[sgprLoopCounterL] // copy loop counter +s_and_b32 s82, s[sgprStaggerU], 0x1f00 +s_lshr_b32 s82, s82, 0x8 +s_and_b32 s83, s[sgprStaggerU], 0xe000 +s_and_b32 s[sgprStaggerU], s[sgprStaggerU], 0xff +s_mov_b32 s80, s[sgprStaggerU] // init staggerU +label_beginStaggerUIter: +s_lshl_b32 s81, s80, s82 // shift by StaggerUStride +s_cmp_ge_u32 s[sgprOrigLoopCounter], s81 // loopCount >= current shift Count +s_cbranch_scc1 label_endStaggerUIter // jump to end +s_lshr_b32 s80, s80, 1 // step down to smaller stagger +s_branch label_beginStaggerUIter // jump to begin +label_endStaggerUIter: +s_sub_u32 s81, s80, 1 // staggerU mask +s_cmp_ge_u32 s80, 1 // if current staggerU >= 1 +s_cselect_b32 s[sgprStaggerUIter], s81, 0 // set Mask +s_cmp_eq_u32 s83, 0x0 +s_cbranch_scc1 label_StaggerUMapping_1 +s_mov_b32 s80, s[sgprWorkGroup0] +s_branch label_staggerInputEnd +label_StaggerUMapping_1: +s_cmp_eq_u32 s83, 0x2000 +s_cbranch_scc1 label_StaggerUMapping_2 +s_mov_b32 s80, s[sgprWorkGroup1] +s_branch label_staggerInputEnd +label_StaggerUMapping_2: +s_cmp_eq_u32 s83, 0x4000 +s_cbranch_scc1 label_StaggerUMapping_3 +s_mov_b32 s80, -0x1 +s_branch label_staggerInputEnd +label_StaggerUMapping_3: +s_cmp_eq_u32 s83, 0x6000 +s_cbranch_scc1 label_StaggerUMapping_4 +s_mul_i32 s81, s[sgprNumWorkGroups0], s[sgprWorkGroup1] +s_add_u32 s80, s80, s81 +s_add_u32 s80, s80, s[sgprWorkGroup0] +s_branch label_staggerInputEnd +label_StaggerUMapping_4: +s_cmp_eq_u32 s83, 0x8000 +s_cbranch_scc1 label_staggerInputEnd +s_mov_b32 s80, -0x1 +s_branch label_staggerInputEnd +label_staggerInputEnd: +s_and_b32 s[sgprStaggerUIter], s[sgprStaggerUIter], s80 // Compute actual stagger start for this tile +s_lshl_b32 s[sgprStaggerUIter], s[sgprStaggerUIter], s82 // shift by StaggerUStride + +/* SRDs += (StaggerUIter) * GlobalReadIncsA+0 */ +s_mul_hi_i32 s81, s[sgprStaggerUIter], s[sgprGlobalReadIncsA+0] // stagger byte offset +s_mul_i32 s80, s[sgprStaggerUIter], s[sgprGlobalReadIncsA+0] // stagger byte offset +s_mul_hi_i32 s[sgprWrapUA+1], s[sgprLoopCounterL], s[sgprGlobalReadIncsA+0] // Number of bytes accessed by the unroll loop +s_mul_i32 s[sgprWrapUA+0], s[sgprLoopCounterL], s[sgprGlobalReadIncsA+0] // Number of bytes accessed by the unroll loop +s_sub_u32 s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0], s[sgprWrapUA+0] // remove one iteration +s_subb_u32 s[sgprWrapUA+1], 0, s[sgprWrapUA+1] // remove one iteration +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 + +/* SRDs += (StaggerUIter) * GlobalReadIncsB+0 */ +s_mul_hi_i32 s81, s[sgprStaggerUIter], s[sgprGlobalReadIncsB+0] // stagger byte offset +s_mul_i32 s80, s[sgprStaggerUIter], s[sgprGlobalReadIncsB+0] // stagger byte offset +s_mul_hi_i32 s[sgprWrapUB+1], s[sgprLoopCounterL], s[sgprGlobalReadIncsB+0] // Number of bytes accessed by the unroll loop +s_mul_i32 s[sgprWrapUB+0], s[sgprLoopCounterL], s[sgprGlobalReadIncsB+0] // Number of bytes accessed by the unroll loop +s_sub_u32 s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0], s[sgprWrapUB+0] // remove one iteration +s_subb_u32 s[sgprWrapUB+1], 0, s[sgprWrapUB+1] // remove one iteration +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +s_add_u32 s[sgprStaggerUIter], s[sgprStaggerUIter], 2 // Subtract (PGR-1); StaggerUIter now contains target iteration to wrap +/* local read addresses: init pointers a */ + +/* localReadInitPointers */ +/* local read addresses: init pointers b */ + +/* localReadInitPointers */ + +/* prefetch: global -> local */ +s_cmp_eq_u32 s[sgprLoopCounterL], 0 // at last iteration? +s_cbranch_scc1 label_ShadowInitStart // skip to ShadowInitStart iter b/c numIter==0 +buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 +buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 +buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 +buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 +buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 +buffer_load_dwordx4 v[vgprValuA_X0_I0+0:vgprValuA_X0_I0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0+0:vgprValuA_X2_I0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X0_I0+4:vgprValuA_X0_I0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0+4:vgprValuA_X2_I0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X0_I0+8:vgprValuA_X0_I0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0+8:vgprValuA_X2_I0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X0_I0+12:vgprValuA_X0_I0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_3_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0+12:vgprValuA_X2_I0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 + +/* global read inc A loopL */ +s_add_u32 s82, s[sgprLoopCounterL], 1 // remove pf(1) +s_cmp_eq_u32 s[sgprStaggerUIter], s82 // Is this wrapIter? (pf) +s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 + +/* global read inc B loopL */ +s_add_u32 s82, s[sgprLoopCounterL], 1 // remove pf(1) +s_cmp_eq_u32 s[sgprStaggerUIter], s82 // Is this wrapIter? (pf) +s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 + +/******************************************/ +/* End setupNewTile */ +/******************************************/ +label_ShadowInitStart: +s_mov_b32 s[sgprSrdD+0], s[sgprAddressD+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdD+1], s[sgprAddressD+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdD+2], 0x80000000 +s_mov_b32 s[sgprSrdD+3], Srd127_96 // Set bits 127_96 in post-loop SRD + +s_mov_b32 s[sgprSrdC+0], s[sgprAddressC+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdC+1], s[sgprAddressC+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdC+2], 0x80000000 +s_mov_b32 s[sgprSrdC+3], Srd127_96 // Set bits 127_96 in post-loop SRD + + +s_mul_i32 s82, MT1, s[sgprWorkGroup1] // <- wg1*MT1 +s_mul_hi_u32 s81, s82, s[sgprStrideC1J] // ScaleC s82 by Stride +s_mul_i32 s80, s82, s[sgprStrideC1J] // ScaleC s82 by Stride +s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeC] // scale by bpe +s_add_u32 s[sgprSrdC+0], s[sgprAddressC+0], s80 // add lo to SRD +s_addc_u32 s[sgprSrdC+1], s[sgprAddressC+1], s81 // add hi to SRD +s_mul_hi_u32 s81, s82, s[sgprStrideD1J] // ScaleD s82 by Stride +s_mul_i32 s80, s82, s[sgprStrideD1J] // ScaleD s82 by Stride +s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeD] // scale by bpe +s_add_u32 s[sgprSrdD+0], s[sgprAddressD+0], s80 // add lo to SRD +s_addc_u32 s[sgprSrdD+1], s[sgprAddressD+1], s81 // add hi to SRD + +s_mul_hi_u32 s81, s[sgprWorkGroup2], s[sgprStrideCK] // ScaleC s[sgprWorkGroup2] by Stride +s_mul_i32 s80, s[sgprWorkGroup2], s[sgprStrideCK] // ScaleC s[sgprWorkGroup2] by Stride +s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeC] // scale by bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s80 // add lo to SRD +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], s81 // add hi to SRD +s_mul_hi_u32 s81, s[sgprWorkGroup2], s[sgprStrideDK] // ScaleD s[sgprWorkGroup2] by Stride +s_mul_i32 s80, s[sgprWorkGroup2], s[sgprStrideDK] // ScaleD s[sgprWorkGroup2] by Stride +s_lshl_b64 s[80:81], s[80:81], s[sgprGSULog2BpeD] // scale by bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s80 // add lo to SRD +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], s81 // add hi to SRD + +s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? +s_cbranch_scc1 label_GSU_2 // branch if GSU == 1 +// GSU Output Buffer offset: Free0 + (Free1-1)*StrideC1J + (Free2-1)*StrideCK * GSUIdx * bpe%s +s_mul_hi_u32 s81, s[sgprSizesFree+0], s[sgprGSUSumIdx] // Free0 +s_mul_i32 s80, s[sgprSizesFree+0], s[sgprGSUSumIdx] // Free0 +s_sub_u32 s82, s[sgprSizesFree+1], 1 // Free1 +s_mul_i32 s82, s82, s[sgprGSUSumIdx] // Free1 +s_mul_hi_u32 s83, s82, s[sgprStrideC1J] // Free1 +s_mul_i32 s82, s82, s[sgprStrideC1J] // Free1 +s_add_u32 s80, s80, s82 // Free1 +s_addc_u32 s81, s81, s83 // Free1 +s_sub_u32 s82, s[sgprSizesFree+2], 1 // Free2 +s_mul_i32 s82, s82, s[sgprGSUSumIdx] // Free2 +s_mul_hi_u32 s83, s82, s[sgprStrideCK] // Free2 +s_mul_i32 s82, s82, s[sgprStrideCK] // Free2 +s_add_u32 s80, s80, s82 // Free2 +s_addc_u32 s81, s81, s83 // Free2 +s_lshl_b64 s[80:81], s[80:81], 2 // scale by bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s80 // add lo GSU offset to SRD +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], s81 // add hi GSU offset to SRD +label_GSU_2: +.set sgprGSULog2BpeC, UNDEF + +/* initC: remove ValuC vgpr buffer [0...0) from pool */ + +/* initC: remove acc vgpr buffer [0...256) from pool */ + +/* initC: remove ValuA/B vgpr buffer [0...160) from pool */ +v_accvgpr_write acc0, 0x0 // initC +v_accvgpr_write acc1, 0x0 // initC +v_accvgpr_write acc2, 0x0 // initC +v_accvgpr_write acc3, 0x0 // initC +v_accvgpr_write acc4, 0x0 // initC +v_accvgpr_write acc5, 0x0 // initC +v_accvgpr_write acc6, 0x0 // initC +v_accvgpr_write acc7, 0x0 // initC +v_accvgpr_write acc8, 0x0 // initC +v_accvgpr_write acc9, 0x0 // initC +v_accvgpr_write acc10, 0x0 // initC +v_accvgpr_write acc11, 0x0 // initC +v_accvgpr_write acc12, 0x0 // initC +v_accvgpr_write acc13, 0x0 // initC +v_accvgpr_write acc14, 0x0 // initC +v_accvgpr_write acc15, 0x0 // initC +v_accvgpr_write acc16, 0x0 // initC +v_accvgpr_write acc17, 0x0 // initC +v_accvgpr_write acc18, 0x0 // initC +v_accvgpr_write acc19, 0x0 // initC +v_accvgpr_write acc20, 0x0 // initC +v_accvgpr_write acc21, 0x0 // initC +v_accvgpr_write acc22, 0x0 // initC +v_accvgpr_write acc23, 0x0 // initC +v_accvgpr_write acc24, 0x0 // initC +v_accvgpr_write acc25, 0x0 // initC +v_accvgpr_write acc26, 0x0 // initC +v_accvgpr_write acc27, 0x0 // initC +v_accvgpr_write acc28, 0x0 // initC +v_accvgpr_write acc29, 0x0 // initC +v_accvgpr_write acc30, 0x0 // initC +v_accvgpr_write acc31, 0x0 // initC +v_accvgpr_write acc32, 0x0 // initC +v_accvgpr_write acc33, 0x0 // initC +v_accvgpr_write acc34, 0x0 // initC +v_accvgpr_write acc35, 0x0 // initC +v_accvgpr_write acc36, 0x0 // initC +v_accvgpr_write acc37, 0x0 // initC +v_accvgpr_write acc38, 0x0 // initC +v_accvgpr_write acc39, 0x0 // initC +v_accvgpr_write acc40, 0x0 // initC +v_accvgpr_write acc41, 0x0 // initC +v_accvgpr_write acc42, 0x0 // initC +v_accvgpr_write acc43, 0x0 // initC +v_accvgpr_write acc44, 0x0 // initC +v_accvgpr_write acc45, 0x0 // initC +v_accvgpr_write acc46, 0x0 // initC +v_accvgpr_write acc47, 0x0 // initC +v_accvgpr_write acc48, 0x0 // initC +v_accvgpr_write acc49, 0x0 // initC +v_accvgpr_write acc50, 0x0 // initC +v_accvgpr_write acc51, 0x0 // initC +v_accvgpr_write acc52, 0x0 // initC +v_accvgpr_write acc53, 0x0 // initC +v_accvgpr_write acc54, 0x0 // initC +v_accvgpr_write acc55, 0x0 // initC +v_accvgpr_write acc56, 0x0 // initC +v_accvgpr_write acc57, 0x0 // initC +v_accvgpr_write acc58, 0x0 // initC +v_accvgpr_write acc59, 0x0 // initC +v_accvgpr_write acc60, 0x0 // initC +v_accvgpr_write acc61, 0x0 // initC +v_accvgpr_write acc62, 0x0 // initC +v_accvgpr_write acc63, 0x0 // initC +v_accvgpr_write acc64, 0x0 // initC +v_accvgpr_write acc65, 0x0 // initC +v_accvgpr_write acc66, 0x0 // initC +v_accvgpr_write acc67, 0x0 // initC +v_accvgpr_write acc68, 0x0 // initC +v_accvgpr_write acc69, 0x0 // initC +v_accvgpr_write acc70, 0x0 // initC +v_accvgpr_write acc71, 0x0 // initC +v_accvgpr_write acc72, 0x0 // initC +v_accvgpr_write acc73, 0x0 // initC +v_accvgpr_write acc74, 0x0 // initC +v_accvgpr_write acc75, 0x0 // initC +v_accvgpr_write acc76, 0x0 // initC +v_accvgpr_write acc77, 0x0 // initC +v_accvgpr_write acc78, 0x0 // initC +v_accvgpr_write acc79, 0x0 // initC +v_accvgpr_write acc80, 0x0 // initC +v_accvgpr_write acc81, 0x0 // initC +v_accvgpr_write acc82, 0x0 // initC +v_accvgpr_write acc83, 0x0 // initC +v_accvgpr_write acc84, 0x0 // initC +v_accvgpr_write acc85, 0x0 // initC +v_accvgpr_write acc86, 0x0 // initC +v_accvgpr_write acc87, 0x0 // initC +v_accvgpr_write acc88, 0x0 // initC +v_accvgpr_write acc89, 0x0 // initC +v_accvgpr_write acc90, 0x0 // initC +v_accvgpr_write acc91, 0x0 // initC +v_accvgpr_write acc92, 0x0 // initC +v_accvgpr_write acc93, 0x0 // initC +v_accvgpr_write acc94, 0x0 // initC +v_accvgpr_write acc95, 0x0 // initC +v_accvgpr_write acc96, 0x0 // initC +v_accvgpr_write acc97, 0x0 // initC +v_accvgpr_write acc98, 0x0 // initC +v_accvgpr_write acc99, 0x0 // initC +v_accvgpr_write acc100, 0x0 // initC +v_accvgpr_write acc101, 0x0 // initC +v_accvgpr_write acc102, 0x0 // initC +v_accvgpr_write acc103, 0x0 // initC +v_accvgpr_write acc104, 0x0 // initC +v_accvgpr_write acc105, 0x0 // initC +v_accvgpr_write acc106, 0x0 // initC +v_accvgpr_write acc107, 0x0 // initC +v_accvgpr_write acc108, 0x0 // initC +v_accvgpr_write acc109, 0x0 // initC +v_accvgpr_write acc110, 0x0 // initC +v_accvgpr_write acc111, 0x0 // initC +v_accvgpr_write acc112, 0x0 // initC +v_accvgpr_write acc113, 0x0 // initC +v_accvgpr_write acc114, 0x0 // initC +v_accvgpr_write acc115, 0x0 // initC +v_accvgpr_write acc116, 0x0 // initC +v_accvgpr_write acc117, 0x0 // initC +v_accvgpr_write acc118, 0x0 // initC +v_accvgpr_write acc119, 0x0 // initC +v_accvgpr_write acc120, 0x0 // initC +v_accvgpr_write acc121, 0x0 // initC +v_accvgpr_write acc122, 0x0 // initC +v_accvgpr_write acc123, 0x0 // initC +v_accvgpr_write acc124, 0x0 // initC +v_accvgpr_write acc125, 0x0 // initC +v_accvgpr_write acc126, 0x0 // initC +v_accvgpr_write acc127, 0x0 // initC +v_accvgpr_write acc128, 0x0 // initC +v_accvgpr_write acc129, 0x0 // initC +v_accvgpr_write acc130, 0x0 // initC +v_accvgpr_write acc131, 0x0 // initC +v_accvgpr_write acc132, 0x0 // initC +v_accvgpr_write acc133, 0x0 // initC +v_accvgpr_write acc134, 0x0 // initC +v_accvgpr_write acc135, 0x0 // initC +v_accvgpr_write acc136, 0x0 // initC +v_accvgpr_write acc137, 0x0 // initC +v_accvgpr_write acc138, 0x0 // initC +v_accvgpr_write acc139, 0x0 // initC +v_accvgpr_write acc140, 0x0 // initC +v_accvgpr_write acc141, 0x0 // initC +v_accvgpr_write acc142, 0x0 // initC +v_accvgpr_write acc143, 0x0 // initC +v_accvgpr_write acc144, 0x0 // initC +v_accvgpr_write acc145, 0x0 // initC +v_accvgpr_write acc146, 0x0 // initC +v_accvgpr_write acc147, 0x0 // initC +v_accvgpr_write acc148, 0x0 // initC +v_accvgpr_write acc149, 0x0 // initC +v_accvgpr_write acc150, 0x0 // initC +v_accvgpr_write acc151, 0x0 // initC +v_accvgpr_write acc152, 0x0 // initC +v_accvgpr_write acc153, 0x0 // initC +v_accvgpr_write acc154, 0x0 // initC +v_accvgpr_write acc155, 0x0 // initC +v_accvgpr_write acc156, 0x0 // initC +v_accvgpr_write acc157, 0x0 // initC +v_accvgpr_write acc158, 0x0 // initC +v_accvgpr_write acc159, 0x0 // initC +v_accvgpr_write acc160, 0x0 // initC +v_accvgpr_write acc161, 0x0 // initC +v_accvgpr_write acc162, 0x0 // initC +v_accvgpr_write acc163, 0x0 // initC +v_accvgpr_write acc164, 0x0 // initC +v_accvgpr_write acc165, 0x0 // initC +v_accvgpr_write acc166, 0x0 // initC +v_accvgpr_write acc167, 0x0 // initC +v_accvgpr_write acc168, 0x0 // initC +v_accvgpr_write acc169, 0x0 // initC +v_accvgpr_write acc170, 0x0 // initC +v_accvgpr_write acc171, 0x0 // initC +v_accvgpr_write acc172, 0x0 // initC +v_accvgpr_write acc173, 0x0 // initC +v_accvgpr_write acc174, 0x0 // initC +v_accvgpr_write acc175, 0x0 // initC +v_accvgpr_write acc176, 0x0 // initC +v_accvgpr_write acc177, 0x0 // initC +v_accvgpr_write acc178, 0x0 // initC +v_accvgpr_write acc179, 0x0 // initC +v_accvgpr_write acc180, 0x0 // initC +v_accvgpr_write acc181, 0x0 // initC +v_accvgpr_write acc182, 0x0 // initC +v_accvgpr_write acc183, 0x0 // initC +v_accvgpr_write acc184, 0x0 // initC +v_accvgpr_write acc185, 0x0 // initC +v_accvgpr_write acc186, 0x0 // initC +v_accvgpr_write acc187, 0x0 // initC +v_accvgpr_write acc188, 0x0 // initC +v_accvgpr_write acc189, 0x0 // initC +v_accvgpr_write acc190, 0x0 // initC +v_accvgpr_write acc191, 0x0 // initC +v_accvgpr_write acc192, 0x0 // initC +v_accvgpr_write acc193, 0x0 // initC +v_accvgpr_write acc194, 0x0 // initC +v_accvgpr_write acc195, 0x0 // initC +v_accvgpr_write acc196, 0x0 // initC +v_accvgpr_write acc197, 0x0 // initC +v_accvgpr_write acc198, 0x0 // initC +v_accvgpr_write acc199, 0x0 // initC +v_accvgpr_write acc200, 0x0 // initC +v_accvgpr_write acc201, 0x0 // initC +v_accvgpr_write acc202, 0x0 // initC +v_accvgpr_write acc203, 0x0 // initC +v_accvgpr_write acc204, 0x0 // initC +v_accvgpr_write acc205, 0x0 // initC +v_accvgpr_write acc206, 0x0 // initC +v_accvgpr_write acc207, 0x0 // initC +v_accvgpr_write acc208, 0x0 // initC +v_accvgpr_write acc209, 0x0 // initC +v_accvgpr_write acc210, 0x0 // initC +v_accvgpr_write acc211, 0x0 // initC +v_accvgpr_write acc212, 0x0 // initC +v_accvgpr_write acc213, 0x0 // initC +v_accvgpr_write acc214, 0x0 // initC +v_accvgpr_write acc215, 0x0 // initC +v_accvgpr_write acc216, 0x0 // initC +v_accvgpr_write acc217, 0x0 // initC +v_accvgpr_write acc218, 0x0 // initC +v_accvgpr_write acc219, 0x0 // initC +v_accvgpr_write acc220, 0x0 // initC +v_accvgpr_write acc221, 0x0 // initC +v_accvgpr_write acc222, 0x0 // initC +v_accvgpr_write acc223, 0x0 // initC +v_accvgpr_write acc224, 0x0 // initC +v_accvgpr_write acc225, 0x0 // initC +v_accvgpr_write acc226, 0x0 // initC +v_accvgpr_write acc227, 0x0 // initC +v_accvgpr_write acc228, 0x0 // initC +v_accvgpr_write acc229, 0x0 // initC +v_accvgpr_write acc230, 0x0 // initC +v_accvgpr_write acc231, 0x0 // initC +v_accvgpr_write acc232, 0x0 // initC +v_accvgpr_write acc233, 0x0 // initC +v_accvgpr_write acc234, 0x0 // initC +v_accvgpr_write acc235, 0x0 // initC +v_accvgpr_write acc236, 0x0 // initC +v_accvgpr_write acc237, 0x0 // initC +v_accvgpr_write acc238, 0x0 // initC +v_accvgpr_write acc239, 0x0 // initC +v_accvgpr_write acc240, 0x0 // initC +v_accvgpr_write acc241, 0x0 // initC +v_accvgpr_write acc242, 0x0 // initC +v_accvgpr_write acc243, 0x0 // initC +v_accvgpr_write acc244, 0x0 // initC +v_accvgpr_write acc245, 0x0 // initC +v_accvgpr_write acc246, 0x0 // initC +v_accvgpr_write acc247, 0x0 // initC +v_accvgpr_write acc248, 0x0 // initC +v_accvgpr_write acc249, 0x0 // initC +v_accvgpr_write acc250, 0x0 // initC +v_accvgpr_write acc251, 0x0 // initC +v_accvgpr_write acc252, 0x0 // initC +v_accvgpr_write acc253, 0x0 // initC +v_accvgpr_write acc254, 0x0 // initC +v_accvgpr_write acc255, 0x0 // initC +s_cmp_eq_u32 s[sgprLoopCounterL], 0 // at last iteration? + +/* after InitC, skip to end of prefetch last iter if numIter==0 */ +s_cbranch_scc0 label_NoBranch_5L94Q2PNAOS4OEC5_0 // Only branch on scc1 +s_getpc_b64 s[80:81] // addr of next instr +s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset +s_add_u32 s80, s80, s82 // add target branch offset +s_addc_u32 s81, s81, 0 // add high and carry +s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd +label_NoBranch_5L94Q2PNAOS4OEC5_0: +s_waitcnt vmcnt(8) +/* local write a */ + +/* local write b */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 + +/* local write swap a */ + +/* local write swap b */ +s_cmp_eq_u32 s[sgprLoopCounterL], 0x1 // PGR=2 but only 1 loop +s_cbranch_scc1 label_skipPGR2_0 // PGR=2 but only 1 loop +buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 +buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 +buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 +buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 +buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 + +label_skipPGR2_0: +s_waitcnt lgkmcnt(0) // 0prefetch wait for local write +// Skip force waitcnt0 +s_barrier + +/* local read prefetch a */ + +/* local read prefetch b */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 + +/* local read inc a */ +/* N/A, lro->32 */ +/* self.localReadDoCntA 1 self.localReadDoCntB 1 */ + +/* local read inc b */ +/* N/A, lro->32 */ +/* self.localReadDoCntA 1 self.localReadDoCntB 1 */ + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 + +/******************************************/ +/* Unrolled Loop(s) - Begin */ +/******************************************/ +label_openLoopL: +s_cmp_eq_u32 s[sgprLoopCounterL], 0x1 // LoopCounterL < EndCounter +s_cbranch_scc1 label_LoopEndL_odd_NoLoadLoop +s_cmp_le_u32 s[sgprLoopCounterL], 0x2 // LoopCounterL < EndCounter +s_cbranch_scc1 label_LoopEndL_even // do not enter LoopL +label_LoopBeginL: + +/******************************************/ +/* Unrolled Loop 1/2 - Begin */ +/******************************************/ + +s_waitcnt vmcnt(8) + +/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ + +/* iter 0 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:1 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+0:vgprValuA_X0_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+0:vgprValuA_X2_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc B loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:3 */ +s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:4 */ +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:6 */ +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:7 */ +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:9 */ +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:10 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:11 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:12 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:13 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:14 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:15 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:16 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:17 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:18 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:19 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:20 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:21 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:22 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:23 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:24 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:25 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:26 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:27 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:28 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:29 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +/* mfmaIndex:30 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:31 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:32 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+4:vgprValuA_X0_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+4:vgprValuA_X2_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:33 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:34 */ +/* schedule remaining localreads for 1LDSB */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:35 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:36 */ +ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:37 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:38 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:39 */ +ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:40 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:41 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:42 */ +ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:43 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:44 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:45 */ +ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:46 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:47 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:48 */ +ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:49 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:50 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:51 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:52 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:53 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:54 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:55 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:56 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:57 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:58 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:59 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:60 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:61 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:62 */ +/* 1 LDS buffer: read-sync-write */ +s_waitcnt lgkmcnt(0) +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:63 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 1 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:64 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:65 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+8:vgprValuA_X0_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+8:vgprValuA_X2_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:66 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:67 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:68 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:69 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:70 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:71 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:72 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:73 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:74 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:75 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:76 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:77 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:78 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:79 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:80 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:81 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:82 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:83 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:84 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:85 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:86 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:87 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:88 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:89 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:90 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:91 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:92 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:93 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:94 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:95 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:96 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+12:vgprValuA_X0_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_3_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+12:vgprValuA_X2_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc A loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:97 */ +s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:98 */ +s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:99 */ +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:100 */ +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:101 */ +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:102 */ +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:103 */ +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:104 */ +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:105 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:106 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:107 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:108 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:109 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:110 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:111 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:112 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:113 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:114 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:115 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:116 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:117 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:118 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:119 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:120 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:121 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:122 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:123 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:124 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:125 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:126 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:127 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:128 */ +buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:129 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:130 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:131 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:132 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:133 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:134 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:135 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:136 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:137 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:138 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:139 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:140 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:141 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:142 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:143 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:144 */ +buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:145 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:146 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:147 */ +/* sched write - iter 2 writesPerItem=1 */ +/* sched write - iter 2 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:148 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:149 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:150 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:151 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:152 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:153 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:154 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:155 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:156 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:157 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:158 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:159 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:160 */ +buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:161 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:162 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:163 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:164 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:165 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:166 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:167 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:168 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:169 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:170 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:171 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:172 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:173 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:174 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:175 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:176 */ +buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:177 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:178 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:179 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:180 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:181 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:182 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:183 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:184 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:185 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:186 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:187 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:188 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:189 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:190 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:191 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ + +/* iter 3 (swap and reset local write pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:192 */ +buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:193 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:194 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:195 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:196 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:197 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:198 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:199 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:200 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:201 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:202 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:203 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:204 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:205 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:206 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:207 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:208 */ +buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:209 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:210 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:211 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:212 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:213 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:214 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:215 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:216 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:217 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:218 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:219 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:220 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:221 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:222 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:223 */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:224 */ +buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:225 */ +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:226 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:227 */ +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:228 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:229 */ +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:230 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:231 */ +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:232 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:233 */ +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:234 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:235 */ +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:236 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:237 */ +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:238 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:239 */ +ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:240 */ +buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:241 */ +ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:242 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:243 */ +ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:244 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:245 */ +ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:246 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:247 */ +ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:248 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:249 */ +ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:250 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:251 */ +ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:252 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:253 */ +ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:254 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:255 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=1 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/******************************************/ +/* Unrolled Loop - End */ +/******************************************/ + +/* closeLoop loopL finalLoop=1 tailLoop=0 */ +s_sub_u32 s[sgprLoopCounterL], s[sgprLoopCounterL], 1 // dec counterL +s_cmp_eq_i32 s[sgprLoopCounterL], 0x2 // counterL==1 +s_cbranch_scc1 label_LoopEndL_odd // to End + +/******************************************/ +/* Unrolled Loop 2/2 - Begin */ +/******************************************/ + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 + +s_waitcnt vmcnt(8) + +/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ + +/* iter 0 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:1 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+0:vgprValuA_X0_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+0:vgprValuA_X2_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc B loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:3 */ +s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:4 */ +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:6 */ +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:7 */ +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:9 */ +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:10 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:11 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:12 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:13 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:14 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:15 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:16 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:17 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:18 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:19 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:20 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:21 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:22 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:23 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:24 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:25 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:26 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:27 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:28 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:29 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:30 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:31 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:32 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+4:vgprValuA_X0_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+4:vgprValuA_X2_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:33 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:34 */ +/* schedule remaining localreads for 1LDSB */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:35 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:36 */ +ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:37 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:38 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:39 */ +ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:40 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:41 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:42 */ +ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:43 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:44 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:45 */ +ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:46 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:47 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:48 */ +ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:49 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:50 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:51 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:52 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:53 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:54 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:55 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:56 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:57 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:58 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:59 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:60 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:61 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:62 */ +/* 1 LDS buffer: read-sync-write */ +s_waitcnt lgkmcnt(0) +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:63 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 1 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:64 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:65 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+8:vgprValuA_X0_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_2_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+8:vgprValuA_X2_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:66 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:67 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:68 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:69 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:70 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:71 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:72 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:73 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:74 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:75 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:76 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:77 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:78 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:79 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:80 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:81 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:82 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:83 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:84 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:85 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:86 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:87 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:88 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:89 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:90 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:91 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:92 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:93 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:94 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:95 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:96 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+12:vgprValuA_X0_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_3_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+12:vgprValuA_X2_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc A loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:97 */ +s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:98 */ +s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:99 */ +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:100 */ +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:101 */ +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:102 */ +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:103 */ +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:104 */ +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:105 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:106 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:107 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:108 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:109 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:110 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:111 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:112 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:113 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:114 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:115 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:116 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:117 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:118 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:119 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:120 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:121 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:122 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:123 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:124 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:125 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:126 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:127 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:128 */ +buffer_load_dwordx4 v[vgprG2LB+0:vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:129 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:130 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:131 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:132 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:133 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:134 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:135 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:136 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:137 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:138 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:139 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:140 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:141 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:142 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:143 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:144 */ +buffer_load_dwordx4 v[vgprG2LB+4:vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:145 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:146 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:147 */ +/* sched write - iter 2 writesPerItem=1 */ +/* sched write - iter 2 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:148 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:149 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:150 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:151 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:152 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:153 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:154 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:155 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:156 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:157 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:158 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:159 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:160 */ +buffer_load_dwordx4 v[vgprG2LB+8:vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // G -> Reg 0_0_2_0 +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:161 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:162 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:163 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:164 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:165 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:166 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:167 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:168 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:169 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:170 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:171 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:172 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:173 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:174 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:175 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:176 */ +buffer_load_dwordx4 v[vgprG2LB+12:vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // G -> Reg 0_0_3_0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:177 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:178 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:179 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:180 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:181 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:182 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:183 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:184 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:185 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:186 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:187 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:188 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:189 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:190 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:191 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ + +/* iter 3 (swap and reset local write pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:192 */ +buffer_load_dwordx4 v[vgprG2LB+16:vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // G -> Reg 0_0_4_0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:193 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:194 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:195 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:196 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:197 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:198 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:199 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:200 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:201 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:202 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:203 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:204 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:205 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:206 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:207 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:208 */ +buffer_load_dwordx4 v[vgprG2LB+20:vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // G -> Reg 0_0_5_0 +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:209 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:210 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:211 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:212 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:213 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:214 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:215 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:216 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:217 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:218 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:219 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:220 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:221 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:222 */ +/* local write swap offsets a */ + +/* local write swap offsets b */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:223 */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:224 */ +buffer_load_dwordx4 v[vgprG2LB+24:vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // G -> Reg 0_0_6_0 +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:225 */ +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:226 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:227 */ +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:228 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:229 */ +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:230 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:231 */ +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:232 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:233 */ +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:234 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:235 */ +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:236 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:237 */ +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:238 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:239 */ +ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:240 */ +buffer_load_dwordx4 v[vgprG2LB+28:vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // G -> Reg 0_0_7_0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:241 */ +ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:242 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:243 */ +ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:244 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:245 */ +ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:246 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:247 */ +ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:248 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:249 */ +ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:250 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:251 */ +ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:252 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:253 */ +ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:254 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:255 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=1 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 +/******************************************/ +/* Unrolled Loop - End */ +/******************************************/ + +/* closeLoop loopL finalLoop=1 tailLoop=0 */ +s_sub_u32 s[sgprLoopCounterL], s[sgprLoopCounterL], 1 // dec counterL +s_cmp_eq_i32 s[sgprLoopCounterL], 0x2 // counterL==2 +s_cbranch_scc0 label_LoopBeginL // restart LoopL +label_LoopEndL_even: + +/* Before NLL: Check VGPR.checkin for INT8 LW */ + +/******************************************/ +/* Ord. NoGlobalLoadLoop - Begin */ +/******************************************/ +s_waitcnt vmcnt(8) +/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ + +/* iter 0 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:1 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+0:vgprValuA_X0_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+0:vgprValuA_X2_I0_1+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc B loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:3 */ +s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:4 */ +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:6 */ +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:7 */ +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:9 */ +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:10 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:11 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:12 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:13 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:14 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:15 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:16 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:17 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:18 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:19 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:20 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:21 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:22 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:23 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:24 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:25 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:26 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:27 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:28 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:29 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +/* mfmaIndex:30 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:31 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:32 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+4:vgprValuA_X0_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+4:vgprValuA_X2_I0_1+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:33 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:34 */ +/* schedule remaining localreads for 1LDSB */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:35 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:36 */ +ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:37 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:38 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:39 */ +ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:40 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:41 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:42 */ +ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:43 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:44 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:45 */ +ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:46 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:47 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:48 */ +ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:49 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:50 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:51 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:52 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:53 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:54 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:55 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:56 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:57 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:58 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:59 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:60 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:61 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:62 */ +/* 1 LDS buffer: read-sync-write */ +s_waitcnt lgkmcnt(0) +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:63 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 1 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:64 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:65 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+8:vgprValuA_X0_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+8:vgprValuA_X2_I0_1+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:66 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:67 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:68 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:69 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:70 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:71 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:72 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:73 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:74 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:75 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:76 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:77 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:78 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:79 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:80 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:81 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:82 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:83 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:84 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:85 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:86 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:87 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:88 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:89 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:90 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:91 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:92 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:93 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:94 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:95 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:96 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_1+12:vgprValuA_X0_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_1+12:vgprValuA_X2_I0_1+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc A loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:97 */ +s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:98 */ +s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:99 */ +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:100 */ +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:101 */ +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:102 */ +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:103 */ +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:104 */ +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:105 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:106 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:107 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:108 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:109 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:110 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:111 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:112 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:113 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:114 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:115 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:116 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:117 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:118 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:119 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:120 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:121 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:122 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:123 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:124 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:125 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:126 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:127 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:128 */ +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:129 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:130 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:131 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:132 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:133 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:134 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:135 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:136 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:137 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:138 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:139 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:140 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:141 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:142 */ +s_waitcnt vmcnt(10) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:143 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:144 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:145 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:146 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:147 */ +/* sched write - iter 2 writesPerItem=1 */ +/* sched write - iter 2 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:148 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:149 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:150 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:151 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:152 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:153 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:154 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:155 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:156 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:157 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:158 */ +s_waitcnt vmcnt(9) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:159 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:160 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:161 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:162 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:163 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:164 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:165 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:166 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:167 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:168 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:169 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:170 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:171 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:172 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:173 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:174 */ +s_waitcnt vmcnt(8) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:175 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:176 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:177 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:178 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:179 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:180 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:181 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:182 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:183 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:184 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:185 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:186 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:187 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:188 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:189 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:190 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:191 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ + +/* iter 3 (swap and reset local write pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:192 */ +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:193 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:194 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:195 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:196 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:197 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:198 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:199 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:200 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:201 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:202 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:203 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:204 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:205 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:206 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:207 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:208 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:209 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:210 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:211 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:212 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:213 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:214 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:215 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:216 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:217 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:218 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:219 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:220 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:221 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:222 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:223 */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:224 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:225 */ +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:226 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:227 */ +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:228 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:229 */ +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:230 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:231 */ +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:232 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:233 */ +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:234 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:235 */ +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:236 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:237 */ +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:238 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:239 */ +ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:240 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:241 */ +ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:242 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:243 */ +ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:244 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:245 */ +ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:246 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:247 */ +ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:248 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:249 */ +ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:250 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:251 */ +ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:252 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:253 */ +ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:254 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:255 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=1 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 + +/******************************************/ +/* Ord. NoLoadLoop - Begin */ +/******************************************/ +s_waitcnt vmcnt(0) + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 + +/* iter 0 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:1 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:2 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:3 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:4 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:5 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:6 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:7 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:9 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:10 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:11 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:12 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:13 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:14 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:15 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:16 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:17 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:18 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:19 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:20 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:21 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:22 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:23 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:24 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:25 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:26 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:27 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:28 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:29 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:30 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:31 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:32 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:33 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:34 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:35 */ +/* sched write - iter 0 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:36 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:37 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:38 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:39 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:40 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:41 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:42 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:43 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:44 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:45 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:46 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:47 */ +/* sched write - iter 0 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:48 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:49 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:50 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:51 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:52 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:53 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:54 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:55 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:56 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:57 */ +ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:58 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:59 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:60 */ +/* sched write - iter 0 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:61 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:62 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:63 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 1 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:64 */ +ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:65 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:66 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:67 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:68 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:69 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:70 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:71 */ +ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:72 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:73 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:74 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:75 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:76 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:77 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:78 */ +ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:79 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:80 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:81 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:82 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:83 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:84 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:85 */ +ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:86 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:87 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:88 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:89 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:90 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:91 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:92 */ +ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:93 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:94 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:95 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:96 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:97 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:98 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:99 */ +ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:100 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:101 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:102 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:103 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:104 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:105 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:106 */ +ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:107 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:108 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:109 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:110 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:111 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:112 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:113 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:114 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:115 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:116 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:117 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:118 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:119 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:120 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:121 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:122 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:123 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:124 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:125 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:126 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:127 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:128 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:129 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:130 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:131 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:132 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:133 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:134 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:135 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:136 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:137 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:138 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:139 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:140 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:141 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:142 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:143 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:144 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:145 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:146 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:147 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:148 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:149 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:150 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:151 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:152 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:153 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:154 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:155 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:156 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:157 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:158 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:159 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:160 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:161 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:162 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:163 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:164 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:165 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:166 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:167 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:168 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:169 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:170 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:171 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:172 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:173 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:174 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:175 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:176 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:177 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:178 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:179 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:180 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:181 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:182 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:183 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:184 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:185 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:186 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:187 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:188 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:189 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:190 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:191 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ + +/* iter 3 (swap and reset local write pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:192 */ +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:193 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:194 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:195 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:196 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:197 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:198 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:199 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:200 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:201 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:202 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:203 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:204 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:205 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:206 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:207 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:208 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:209 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:210 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:211 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:212 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:213 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:214 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:215 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:216 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:217 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:218 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:219 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:220 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:221 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:222 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:223 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:224 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:225 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:226 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:227 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:228 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:229 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:230 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:231 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:232 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:233 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:234 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:235 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:236 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:237 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:238 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:239 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:240 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:241 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:242 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:243 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:244 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:245 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:246 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:247 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:248 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:249 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:250 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:251 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:252 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:253 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:254 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:255 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=1 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +label_Summation_End_OptNLL: +s_cmpk_eq_u32 s[sgprBeta], 0x0 // Beta == 0 +s_cbranch_scc1 label_NoBranch_PAGZHD9H2DI57HHE_0 // Only branch on scc0 +s_getpc_b64 s[80:81] // addr of next instr +s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset +s_add_u32 s80, s80, s82 // add target branch offset +s_addc_u32 s81, s81, 0 // add high and carry +s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd +label_NoBranch_PAGZHD9H2DI57HHE_0: + +s_cmp_eq_u32 s[sgprAlpha], 1.0 // Alpha == 1.0 ? +s_getpc_b64 s[80:81] // addr of next instr +s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset +s_add_u32 s80, s80, s82 // add target branch offset +s_addc_u32 s81, s81, 0 // add high and carry +s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd +label_NoBranch_XQG82FMXJOJL8OIW_0: + +s_and_b32 s80, 255, s[sgprSizeI] // s80 = s[sgprSizeI] % 256 +s_add_u32 s81, -0x1, s[sgprNumWorkGroups0] +s_cmp_ge_u32 s[sgprWorkGroup0], s81 // wg0 >= nwg0-1 ? +s_cselect_b32 s80, s80, 0 // set rMT0 +s_cmpk_gt_u32 s80, 0x0 // rMT0 > 0 +s_cbranch_scc0 label_NoBranch_XQ75AI1RJ5F179IN_0 // Only branch on scc1 +// jump if edges required +s_getpc_b64 s[80:81] // addr of next instr +s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset +s_add_u32 s80, s80, s82 // add target branch offset +s_addc_u32 s81, s81, 0 // add high and carry +s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd +label_NoBranch_XQ75AI1RJ5F179IN_0: +s_and_b32 s80, 255, s[sgprSizeJ] // s80 = s[sgprSizeJ] % 256 +s_add_u32 s81, -0x1, s[sgprNumWorkGroups1] +s_cmp_ge_u32 s[sgprWorkGroup1], s81 // wg1 >= nwg1-1 +s_cselect_b32 s80, s80, 0 // set rMT1 +s_cmpk_gt_u32 s80, 0x0 // rMT1 > 0 +s_cbranch_scc0 label_NoBranch_GFF6GDU2NIUMLQ8E_0 // Only branch on scc1 +// jump if edges required +s_getpc_b64 s[80:81] // addr of next instr +s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset +s_add_u32 s80, s80, s82 // add target branch offset +s_addc_u32 s81, s81, 0 // add high and carry +s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd +label_NoBranch_GFF6GDU2NIUMLQ8E_0: + +s_and_b32 s81, 63, s[sgprSizesSum+0] // s81 = s[sgprSizesSum+0] % 64 +s_cmp_eq_u32 s81, 0x0 // numIterL == 0 +s_cbranch_scc1 label_NoBranch_VEIX0X0UO5PRLIZN_0 // Only branch on scc0 +s_getpc_b64 s[80:81] // addr of next instr +s_add_i32 s82, label_PrefetchGlobalLastIterEnd, 0x4 // target branch offset +s_add_u32 s80, s80, s82 // add target branch offset +s_addc_u32 s81, s81, 0 // add high and carry +s_setpc_b64 s[80:81] // branch to label_PrefetchGlobalLastIterEnd +label_NoBranch_VEIX0X0UO5PRLIZN_0: + +/* endSummation: add vgpr [0...230) to pool */ +/* load store sgprs */ +.set sgprAddressScaleAlphaVec, 48 +.set sgprAddressBias, 50 +.set sgprBiasType, 52 +.set sgprBiasStride, 53 +.set sgpractivationAlpha, 54 +.set sgpractivationBeta, 55 +.set sgprActivationType, 56 +/* Check if custom structure pointer is null */ +s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? +s_cbranch_scc1 label_LoadExternalEpilogueStruct // branch if ArgType == 2 +s_load_dwordx8 s[48:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x58 +s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x78 +s_branch label_LoadExternalEpilogueStructEnd +label_LoadExternalEpilogueStruct: +s_load_dwordx4 s[48:51], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x90 +s_load_dwordx2 s[52:53], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xa0 +s_load_dwordx2 s[54:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xb8 +s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0xc0 +label_LoadExternalEpilogueStructEnd: +.set sgprSrdScaleAlphaVec, 32 +.set sgprSrdBias, 40 + +/* Mapping of Acc register -> C Vgpr register */ +/* computeStoreVgprs */ +v_lshrrev_b32 v4, 6, v[vgprSerial] // v4 = v[vgprSerial] / 64 +v_lshrrev_b32 v5, 2, v4 // v5 = v4 / 4 +v_mul_lo_u32 v5, 0x10, v5 // wave coordination offset 1 +v_and_b32 v1, 63, v[vgprSerial] // v1 = v[vgprSerial] % 64 +v_lshrrev_b32 v1, 4, v1 // v1 = v1 / 16 +v_lshlrev_b32 v1, 0x2, v1 // thread0 * continuous_output +v_add_lshl_u32 v1, v5, v1, 0 // coordination 1 = vwB *(wave_id1 + tid1) +v_mul_lo_u32 v2, v1, s[sgprStrideC1J] // offset 1 +v_mul_lo_u32 v3, v1, s[sgprStrideD1J] // offset 1 +v_and_b32 v0, 3, v4 // v0 = v4 % 4 +v_mul_lo_u32 v0, 0x10, v0 // wave coordination offset 0 +v_and_b32 v5, 15, v[vgprSerial] // v5 = v[vgprSerial] % 16 +v_add_lshl_u32 v0, v5, v0, 2 // coordination 0 = vwA * (wave_id0 + tid0) +s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v0, s8, v0 // coord 0 = (tid0/MI_m)*4 + waveG0*MIB_m + MT0*SG0 +s_mul_i32 s8, 256, s[sgprWorkGroup1] // wgp1 * MT1 +v_add_u32 v1, s8, v1 // coord 1 = (tid0%MI_m) + waveG1*MIB_n + MT1*SG1 + +/******************************************/ +/* Global Write Elements */ +/******************************************/ +s_waitcnt lgkmcnt(0) // wait for 36 bytes of kern args. +s_mov_b32 s[sgprSrdScaleAlphaVec+0], s[sgprAddressScaleAlphaVec+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdScaleAlphaVec+1], s[sgprAddressScaleAlphaVec+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdScaleAlphaVec+3], Srd127_96 // Set bits 127_96 in post-loop SRD +s_cmp_eq_u64 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], 0 // s[AddressScaleAlphaVec] == 0 ? +s_cbranch_scc0 label_ScaleAlphaVecAddrValid // branch if s[AddressScaleAlphaVec] != 0 +s_mov_b32 s[sgprSrdScaleAlphaVec+2], 0 +s_branch label_ScaleAlphaVecAddrValid_End +label_ScaleAlphaVecAddrValid: +s_mov_b32 s[sgprSrdScaleAlphaVec+2], s[sgprSizeI] +label_ScaleAlphaVecAddrValid_End: + +s_mul_i32 s[sgprSrdScaleAlphaVec+2], 0x4, s[sgprSrdScaleAlphaVec+2] // ScaleAlphaVec scaled by BPE +s_add_u32 s8, s[sgprWorkGroup2], 0x1 +s_mul_i32 s8, s[sgprBiasStride], s8 // stride * (wg+1) +s_cmp_eq_u32 s8, 0x0 // bias stride = 0? +s_cselect_b32 s8, s[sgprSizeI], s8 +s_mov_b32 s[sgprSrdBias+0], s[sgprAddressBias+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdBias+1], s[sgprAddressBias+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdBias+3], Srd127_96 // Set bits 127_96 in post-loop SRD +s_cmp_eq_u64 s[sgprAddressBias:sgprAddressBias+1], 0 // s[AddressBias] == 0 ? +s_cbranch_scc0 label_BiasAddrValid // branch if s[AddressBias] != 0 +s_mov_b32 s[sgprSrdBias+2], 0 +s_branch label_BiasAddrValid_End +label_BiasAddrValid: +s_mov_b32 s[sgprSrdBias+2], s8 +label_BiasAddrValid_End: + +label_Load_Biasf32_0: +s_cmpk_lg_u32 s[sgprBiasType], 0 // BiasType != 0 +s_cbranch_scc1 label_Load_Biasbf16_0 // Branch if true + +/******************************************/ +/* Read Bias to LDS */ +/******************************************/ +s_mul_i32 s[sgprSrdBias+2], 0x4, s[sgprSrdBias+2] // scaled by BPE +s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset +s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG +v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG +v_lshlrev_b32 v8, 0x2, v8 // Global bias address scaled by BPE +buffer_load_dword v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias +v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE +s_waitcnt vmcnt(0) // wait for bias load +s_barrier // Wait for all wavefronts +ds_write_b32 v8, v4 offset:0 // store bias +s_branch label_Load_Bias_End // Branch to load bias end +label_Load_Biasbf16_0: +s_cmpk_lg_u32 s[sgprBiasType], 7 // BiasType != 7 +s_cbranch_scc1 label_Load_Bias_End // Branch if true + +/******************************************/ +/* Read Bias to LDS */ +/******************************************/ +s_mul_i32 s[sgprSrdBias+2], 0x2, s[sgprSrdBias+2] // scaled by BPE +s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset +s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG +v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG +v_lshlrev_b32 v8, 0x1, v8 // Global bias address scaled by BPE +buffer_load_short_d16 v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias +v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE +s_waitcnt vmcnt(0) // wait for bias load +s_barrier // Wait for all wavefronts +v_lshlrev_b32 v4, 16, v4 // cvt bf16 to fp32. +ds_write_b32 v8, v4 offset:0 // store bias +s_branch label_Load_Bias_End // Branch to load bias end +label_Load_Bias_End: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW4 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW4 // Branch if true +label_To_Activation_None_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Abs_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Clippedrelu_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Gelu_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Leakyrelu_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Relu_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Sigmoid_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Tanh_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Geluscaling_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_To_Activation_Silu_VW4: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd +label_ActivationSetPCAddrEnd: +label_GW_B0_E0: + +/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b128 v[20:23], v15 offset:0 // load bias +v_lshlrev_b32 v16, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_lshl_u32 v13, v3, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 +v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+32], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+33], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+34], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+35], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+36], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+37], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+38], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+39], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+40], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+41], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+42], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+43], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+44], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+45], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+46], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+47], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+48], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+49], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+50], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+51], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+52], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+53], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+54], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+55], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+56], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+57], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+58], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+59], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+60], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+61], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+62], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+63], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+64], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+65], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+66], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+67], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+68], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+69], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+70], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+71], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+72], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+73], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+74], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+75], acc47 // copy acc to vreg[47] +v_accvgpr_read_b32 v[vgprValuC+76], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+77], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+78], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+79], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+80], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+81], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+82], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+83], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+84], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+85], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+86], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+87], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+88], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+89], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+90], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+91], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+28], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+29], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+31], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+32], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+33], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+34], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+35], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+36], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+37], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+38], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+39], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+40], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+41], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+42], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+43], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+44], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+45], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+46], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+47], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+48], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+49], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+50], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+51], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+52], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+53], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+54], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+55], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+56], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+57], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+58], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+59], acc95 // copy acc to vreg[95] +v_accvgpr_read_b32 v[vgprValuC+60], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+61], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+62], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+63], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+64], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+65], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+66], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+67], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+68], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+69], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+70], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+71], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+72], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+73], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+74], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+75], acc111 // copy acc to vreg[111] +v_accvgpr_read_b32 v[vgprValuC+76], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+77], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+78], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+79], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+80], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+81], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+82], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+83], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+84], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+85], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+86], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+87], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+88], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+89], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+90], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+91], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+28], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+29], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+31], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+32], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+33], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+34], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+35], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+36], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+37], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+38], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+39], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+40], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+41], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+42], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+43], acc143 // copy acc to vreg[143] +v_accvgpr_read_b32 v[vgprValuC+44], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+45], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+46], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+47], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+48], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+49], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+50], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+51], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+52], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+53], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+54], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+55], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+56], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+57], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+58], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+59], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+60], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+61], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+62], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+63], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+64], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+65], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+66], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+67], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+68], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+69], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+70], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+71], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+72], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+73], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+74], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+75], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+76], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+77], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+78], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+79], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+80], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+81], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+82], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+83], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+84], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+85], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+86], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+87], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+88], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+89], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+90], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+91], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+32], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+33], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+34], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+35], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+36], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+37], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+38], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+39], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+40], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+41], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+42], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+43], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+44], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+45], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+46], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+47], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+48], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+49], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+50], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+51], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+52], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+53], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+54], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+55], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+56], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+57], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+58], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+59], acc223 // copy acc to vreg[223] +v_accvgpr_read_b32 v[vgprValuC+60], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+61], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+62], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+63], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+64], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+65], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+66], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+67], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+68], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+69], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+70], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+71], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+72], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+73], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+74], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+75], acc239 // copy acc to vreg[239] +v_accvgpr_read_b32 v[vgprValuC+76], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+77], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+78], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+79], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+80], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+81], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+82], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+83], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+84], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+85], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+86], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+87], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+88], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+89], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+90], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+91], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End // jump to end +label_GW_End: + +s_endpgm // Kernel End + +label_LoopEndL_odd: +.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 + +/* Before NLL: Check VGPR.checkin for INT8 LW */ + +/******************************************/ +/* Ord. NoGlobalLoadLoop - Begin */ +/******************************************/ +s_waitcnt vmcnt(8) +/* Begin Each Unroll: Check VGPR.checkin for INT8 LW */ + +/* iter 0 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:1 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+0:vgprValuA_X0_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // G -> Reg 0_0_0_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+0:vgprValuA_X2_I0_0+0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc B loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:2 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cselect_b32 s80, s[sgprWrapUB+0], s[sgprGlobalReadIncsB+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:3 */ +s_cselect_b32 s81, s[sgprWrapUB+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:4 */ +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:5 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:6 */ +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:7 */ +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:9 */ +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:10 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:11 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:12 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:13 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:14 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:15 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:16 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:17 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:18 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:19 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:20 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:21 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:22 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:23 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:24 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:25 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:26 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:27 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:28 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:29 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +/* mfmaIndex:30 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:31 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:32 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+4:vgprValuA_X0_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+4:vgprValuA_X2_I0_0+4+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // G -> Reg 0_0_1_0 +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:33 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:34 */ +/* schedule remaining localreads for 1LDSB */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:35 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:36 */ +ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:37 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:38 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:39 */ +ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:40 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:41 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:42 */ +ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:43 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:44 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:45 */ +ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:46 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:47 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:48 */ +ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:49 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:50 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:51 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:52 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:53 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:54 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:55 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:56 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:57 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:58 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:59 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:60 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:61 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:62 */ +/* 1 LDS buffer: read-sync-write */ +s_waitcnt lgkmcnt(0) +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:63 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 1 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:64 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:65 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+8:vgprValuA_X0_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+8:vgprValuA_X2_I0_0+8+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // G -> Reg 0_0_1_0 +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:66 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:67 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:68 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:69 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:70 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:71 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:72 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:73 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:74 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:75 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:76 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:77 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:78 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:79 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:80 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:81 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:82 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:83 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:84 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:85 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:86 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:87 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:88 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:89 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:90 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:91 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:92 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:93 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:94 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:95 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:96 */ +buffer_load_dwordx4 v[vgprValuA_X0_I0_0+12:vgprValuA_X0_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // G -> Reg 0_0_1_0 +buffer_load_dwordx4 v[vgprValuA_X2_I0_0+12:vgprValuA_X2_I0_0+12+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // G -> Reg 0_0_1_0 +/* global read inc A loopL */ +s_cmp_eq_u32 s[sgprLoopCounterL], s[sgprStaggerUIter] // Is this the wrapIter? +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:97 */ +s_cselect_b32 s80, s[sgprWrapUA+0], s[sgprGlobalReadIncsA+0] // incLower <- ? +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:98 */ +s_cselect_b32 s81, s[sgprWrapUA+1], 0 // incUpper <- ? +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:99 */ +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:100 */ +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:101 */ +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:102 */ +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:103 */ +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:104 */ +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:105 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:106 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:107 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:108 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:109 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:110 */ +s_waitcnt vmcnt(12) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:111 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:112 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:113 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:114 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:115 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:116 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:117 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:118 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:119 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:120 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:121 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:122 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:123 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:124 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:125 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:126 */ +s_waitcnt vmcnt(11) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:127 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:128 */ +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:129 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:130 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:131 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:132 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:133 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:134 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:135 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:136 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:137 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:138 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:139 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:140 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:141 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:142 */ +s_waitcnt vmcnt(10) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:143 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:144 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:145 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:146 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:147 */ +/* sched write - iter 2 writesPerItem=1 */ +/* sched write - iter 2 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:148 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:149 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:150 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:151 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:152 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:153 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:154 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:155 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:156 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:157 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:158 */ +s_waitcnt vmcnt(9) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:159 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:160 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:161 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:162 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:163 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:164 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:165 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:166 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:167 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:168 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:169 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:170 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:171 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:172 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:173 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:174 */ +s_waitcnt vmcnt(8) // wait for global read before writing to local +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:175 */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:176 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:177 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:178 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:179 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:180 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:181 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:182 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:183 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:184 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:185 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:186 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:187 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:188 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:189 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:190 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:191 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ + +/* iter 3 (swap and reset local write pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:192 */ +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:193 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:194 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:195 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:196 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:197 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:198 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:199 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:200 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:201 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:202 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:203 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:204 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:205 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:206 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:207 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:208 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:209 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:210 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:211 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:212 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:213 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:214 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:215 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:216 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:217 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:218 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:219 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:220 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:221 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:222 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=8 newLW=8 newLR=0 +s_barrier +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:223 */ +ds_read_b128 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+3], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:224 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:225 */ +ds_read_b128 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+3], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:226 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:227 */ +ds_read_b128 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+3], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:228 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:229 */ +ds_read_b128 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+3], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:230 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:231 */ +ds_read_b128 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+3], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:232 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:233 */ +ds_read_b128 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+3], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:234 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:235 */ +ds_read_b128 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+3], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:236 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:237 */ +ds_read_b128 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+3], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:238 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:239 */ +ds_read_b128 v[vgprValuB_X0_I0+32:vgprValuB_X0_I0+32+3], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:240 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:241 */ +ds_read_b128 v[vgprValuB_X0_I0+36:vgprValuB_X0_I0+36+3], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:242 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:243 */ +ds_read_b128 v[vgprValuB_X0_I0+40:vgprValuB_X0_I0+40+3], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:244 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:245 */ +ds_read_b128 v[vgprValuB_X0_I0+44:vgprValuB_X0_I0+44+3], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:246 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:247 */ +ds_read_b128 v[vgprValuB_X0_I0+48:vgprValuB_X0_I0+48+3], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:248 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:249 */ +ds_read_b128 v[vgprValuB_X0_I0+52:vgprValuB_X0_I0+52+3], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:250 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:251 */ +ds_read_b128 v[vgprValuB_X0_I0+56:vgprValuB_X0_I0+56+3], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:252 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:253 */ +ds_read_b128 v[vgprValuB_X0_I0+60:vgprValuB_X0_I0+60+3], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:254 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:255 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=1 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +.set vgprValuA_X0_I0, vgprValuA_X0_I0_0 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_0 + +s_branch label_LoopEndL_odd_NoLoadLoop + +label_LoopEndL_odd_NoLoadLoop: + +/******************************************/ +/* Ord. NoLoadLoop - Begin */ +/******************************************/ +s_waitcnt vmcnt(0) + +/* iter 0 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:0 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:1 */ +ds_read_b128 v[vgprValuB_X2_I0+0:vgprValuB_X2_I0+0+3], v[vgprLocalReadAddrB] offset:64 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:2 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:3 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:4 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:5 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:6 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:7 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:8 */ +ds_read_b128 v[vgprValuB_X2_I0+4:vgprValuB_X2_I0+4+3], v[vgprLocalReadAddrB] offset:2624 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:9 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:10 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:11 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:12 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:13 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:14 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:15 */ +ds_read_b128 v[vgprValuB_X2_I0+8:vgprValuB_X2_I0+8+3], v[vgprLocalReadAddrB] offset:5184 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:16 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:17 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:18 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:19 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:20 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:21 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:22 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+12:vgprValuB_X2_I0+12+3], v[vgprLocalReadAddrB] offset:7744 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:23 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:24 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:25 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:26 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:27 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:28 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:29 */ +/* localReadsVacancy: latencyLeft 2 */ +ds_read_b128 v[vgprValuB_X2_I0+16:vgprValuB_X2_I0+16+3], v[vgprLocalReadAddrB] offset:10304 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:30 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:31 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:32 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:33 */ +/* localReadsVacancy: latencyLeft 2 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:34 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:35 */ +/* sched write - iter 0 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+0+0:vgprValuB_X0_I0+32+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:36 */ +ds_read_b128 v[vgprValuB_X2_I0+20:vgprValuB_X2_I0+20+3], v[vgprLocalReadAddrB] offset:12864 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:37 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:38 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:39 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+0+0:vgprValuB_X0_I0+36+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:40 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:41 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:42 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:43 */ +ds_read_b128 v[vgprValuB_X2_I0+24:vgprValuB_X2_I0+24+3], v[vgprLocalReadAddrB] offset:15424 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+0+0:vgprValuB_X0_I0+40+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:44 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:45 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:46 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:47 */ +/* sched write - iter 0 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+0+0:vgprValuB_X0_I0+44+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:48 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:49 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:50 */ +ds_read_b128 v[vgprValuB_X2_I0+28:vgprValuB_X2_I0+28+3], v[vgprLocalReadAddrB] offset:17984 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:51 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+0+0:vgprValuB_X0_I0+48+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:52 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:53 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:54 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:55 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+0+0:vgprValuB_X0_I0+52+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:56 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:57 */ +ds_read_b128 v[vgprValuB_X2_I0+32:vgprValuB_X2_I0+32+3], v[vgprLocalReadAddrB] offset:20544 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:58 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:59 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+0+0:vgprValuB_X0_I0+56+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:60 */ +/* sched write - iter 0 writesPerItem=1 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:61 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:62 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+8+0+0:vgprValuA_X0_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:63 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+0+0:vgprValuB_X0_I0+60+0+0+1], v[vgprValuA_X0_I0+12+0+0:vgprValuA_X0_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 1 */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:64 */ +ds_read_b128 v[vgprValuB_X2_I0+36:vgprValuB_X2_I0+36+3], v[vgprLocalReadAddrB] offset:23104 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:65 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:66 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:67 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+2+0:vgprValuB_X0_I0+0+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:68 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:69 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:70 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:71 */ +ds_read_b128 v[vgprValuB_X2_I0+40:vgprValuB_X2_I0+40+3], v[vgprLocalReadAddrB] offset:25664 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+4+2+0:vgprValuB_X0_I0+4+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:72 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:73 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:74 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:75 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+8+2+0:vgprValuB_X0_I0+8+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:76 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:77 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:78 */ +ds_read_b128 v[vgprValuB_X2_I0+44:vgprValuB_X2_I0+44+3], v[vgprLocalReadAddrB] offset:28224 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:79 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+12+2+0:vgprValuB_X0_I0+12+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:80 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:81 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:82 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:83 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+16+2+0:vgprValuB_X0_I0+16+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:84 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:85 */ +ds_read_b128 v[vgprValuB_X2_I0+48:vgprValuB_X2_I0+48+3], v[vgprLocalReadAddrB] offset:30784 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:86 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:87 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+20+2+0:vgprValuB_X0_I0+20+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:88 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:89 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:90 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:91 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+24+2+0:vgprValuB_X0_I0+24+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:92 */ +ds_read_b128 v[vgprValuB_X2_I0+52:vgprValuB_X2_I0+52+3], v[vgprLocalReadAddrB] offset:33344 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:93 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:94 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:95 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+28+2+0:vgprValuB_X0_I0+28+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:96 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:97 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:98 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:99 */ +ds_read_b128 v[vgprValuB_X2_I0+56:vgprValuB_X2_I0+56+3], v[vgprLocalReadAddrB] offset:35904 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+32+2+0:vgprValuB_X0_I0+32+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:100 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:101 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:102 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:103 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+36+2+0:vgprValuB_X0_I0+36+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:104 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:105 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:106 */ +ds_read_b128 v[vgprValuB_X2_I0+60:vgprValuB_X2_I0+60+3], v[vgprLocalReadAddrB] offset:38464 // L -> Reg lro=32 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=2 iui=0 +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:107 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+40+2+0:vgprValuB_X0_I0+40+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:108 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:109 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:110 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:111 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+44+2+0:vgprValuB_X0_I0+44+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:112 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:113 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:114 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:115 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+48+2+0:vgprValuB_X0_I0+48+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:116 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:117 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:118 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:119 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+52+2+0:vgprValuB_X0_I0+52+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:120 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:121 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:122 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:123 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+56+2+0:vgprValuB_X0_I0+56+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:124 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+0+2+0:vgprValuA_X0_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:125 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+4+2+0:vgprValuA_X0_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:126 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+8+2+0:vgprValuA_X0_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:127 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+60+2+0:vgprValuB_X0_I0+60+2+0+1], v[vgprValuA_X0_I0+12+2+0:vgprValuA_X0_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=-1 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=-1 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +/* iter 2 (reset local read pointers iteration) (swap local read pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:128 */ +s_waitcnt lgkmcnt(0) // wait for prior local read local write old=0, new=0 newLW=0 newLR=0 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:129 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:130 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:131 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+0+0:vgprValuB_X2_I0+0+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:132 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:133 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:134 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:135 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+0+0:vgprValuB_X2_I0+4+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:136 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:137 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:138 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:139 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+0+0:vgprValuB_X2_I0+8+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:140 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:141 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:142 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:143 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+0+0:vgprValuB_X2_I0+12+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:144 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:145 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:146 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:147 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+0+0:vgprValuB_X2_I0+16+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:148 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:149 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:150 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:151 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+0+0:vgprValuB_X2_I0+20+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:152 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:153 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:154 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:155 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+0+0:vgprValuB_X2_I0+24+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:156 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:157 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:158 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:159 */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+0+0:vgprValuB_X2_I0+28+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:160 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:161 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:162 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:163 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+0+0:vgprValuB_X2_I0+32+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:164 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:165 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:166 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:167 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+0+0:vgprValuB_X2_I0+36+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:168 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:169 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:170 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:171 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+0+0:vgprValuB_X2_I0+40+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:172 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:173 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:174 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:175 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+0+0:vgprValuB_X2_I0+44+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:176 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:177 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:178 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:179 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+0+0:vgprValuB_X2_I0+48+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:180 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:181 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:182 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:183 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+0+0:vgprValuB_X2_I0+52+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:184 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:185 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:186 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:187 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+0+0:vgprValuB_X2_I0+56+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:188 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+0+0+0:vgprValuA_X2_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:189 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+4+0+0:vgprValuA_X2_I0+4+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:190 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+8+0+0:vgprValuA_X2_I0+8+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:191 */ + +/* local read swap offsets a */ + +/* local read swap offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+0+0:vgprValuB_X2_I0+60+0+0+1], v[vgprValuA_X2_I0+12+0+0:vgprValuA_X2_I0+12+0+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=0 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=0 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=0 readsPerIterB=16 */ + +/* iter 3 (swap and reset local write pointers iteration) */ +/* grEndMfmaIndex:18, lwStartMfmaIndex:35, lwEndMfmaIndex:223 */ +/* numMfmaForLR:30, syncPlrMfmaIndex:225 */ +/* mfmaIndex:192 */ +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[0:3] // left value = acc[0+0:3+0] +/* mfmaIndex:193 */ +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[4:7] // left value = acc[4+0:7+0] +/* mfmaIndex:194 */ +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[8:11] // left value = acc[8+0:11+0] +/* mfmaIndex:195 */ +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X2_I0+0+2+0:vgprValuB_X2_I0+0+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[12:15] // left value = acc[12+0:15+0] +/* mfmaIndex:196 */ +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[16:19] // left value = acc[16+0:19+0] +/* mfmaIndex:197 */ +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[20:23] // left value = acc[20+0:23+0] +/* mfmaIndex:198 */ +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[24:27] // left value = acc[24+0:27+0] +/* mfmaIndex:199 */ +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X2_I0+4+2+0:vgprValuB_X2_I0+4+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[28:31] // left value = acc[28+0:31+0] +/* mfmaIndex:200 */ +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[32:35] // left value = acc[32+0:35+0] +/* mfmaIndex:201 */ +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[36:39] // left value = acc[36+0:39+0] +/* mfmaIndex:202 */ +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[40:43] // left value = acc[40+0:43+0] +/* mfmaIndex:203 */ +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X2_I0+8+2+0:vgprValuB_X2_I0+8+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[44:47] // left value = acc[44+0:47+0] +/* mfmaIndex:204 */ +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[48:51] // left value = acc[48+0:51+0] +/* mfmaIndex:205 */ +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[52:55] // left value = acc[52+0:55+0] +/* mfmaIndex:206 */ +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[56:59] // left value = acc[56+0:59+0] +/* mfmaIndex:207 */ +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X2_I0+12+2+0:vgprValuB_X2_I0+12+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[60:63] // left value = acc[60+0:63+0] +/* mfmaIndex:208 */ +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[64:67] // left value = acc[64+0:67+0] +/* mfmaIndex:209 */ +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[68:71] // left value = acc[68+0:71+0] +/* mfmaIndex:210 */ +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[72:75] // left value = acc[72+0:75+0] +/* mfmaIndex:211 */ +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X2_I0+16+2+0:vgprValuB_X2_I0+16+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[76:79] // left value = acc[76+0:79+0] +/* mfmaIndex:212 */ +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[80:83] // left value = acc[80+0:83+0] +/* mfmaIndex:213 */ +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[84:87] // left value = acc[84+0:87+0] +/* mfmaIndex:214 */ +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[88:91] // left value = acc[88+0:91+0] +/* mfmaIndex:215 */ +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X2_I0+20+2+0:vgprValuB_X2_I0+20+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[92:95] // left value = acc[92+0:95+0] +/* mfmaIndex:216 */ +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[96:99] // left value = acc[96+0:99+0] +/* mfmaIndex:217 */ +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[100:103] // left value = acc[100+0:103+0] +/* mfmaIndex:218 */ +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[104:107] // left value = acc[104+0:107+0] +/* mfmaIndex:219 */ +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X2_I0+24+2+0:vgprValuB_X2_I0+24+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[108:111] // left value = acc[108+0:111+0] +/* mfmaIndex:220 */ +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[112:115] // left value = acc[112+0:115+0] +/* mfmaIndex:221 */ +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[116:119] // left value = acc[116+0:119+0] +/* mfmaIndex:222 */ +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[120:123] // left value = acc[120+0:123+0] +/* mfmaIndex:223 */ + +/* local write swap offsets a */ + +/* local write swap offsets b */ +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X2_I0+28+2+0:vgprValuB_X2_I0+28+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[124:127] // left value = acc[124+0:127+0] +/* mfmaIndex:224 */ +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[128:131] // left value = acc[128+0:131+0] +/* mfmaIndex:225 */ +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[132:135] // left value = acc[132+0:135+0] +/* mfmaIndex:226 */ +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[136:139] // left value = acc[136+0:139+0] +/* mfmaIndex:227 */ +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X2_I0+32+2+0:vgprValuB_X2_I0+32+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[140:143] // left value = acc[140+0:143+0] +/* mfmaIndex:228 */ +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[144:147] // left value = acc[144+0:147+0] +/* mfmaIndex:229 */ +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[148:151] // left value = acc[148+0:151+0] +/* mfmaIndex:230 */ +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[152:155] // left value = acc[152+0:155+0] +/* mfmaIndex:231 */ +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X2_I0+36+2+0:vgprValuB_X2_I0+36+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[156:159] // left value = acc[156+0:159+0] +/* mfmaIndex:232 */ +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[160:163] // left value = acc[160+0:163+0] +/* mfmaIndex:233 */ +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[164:167] // left value = acc[164+0:167+0] +/* mfmaIndex:234 */ +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[168:171] // left value = acc[168+0:171+0] +/* mfmaIndex:235 */ +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X2_I0+40+2+0:vgprValuB_X2_I0+40+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[172:175] // left value = acc[172+0:175+0] +/* mfmaIndex:236 */ +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[176:179] // left value = acc[176+0:179+0] +/* mfmaIndex:237 */ +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[180:183] // left value = acc[180+0:183+0] +/* mfmaIndex:238 */ +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[184:187] // left value = acc[184+0:187+0] +/* mfmaIndex:239 */ +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X2_I0+44+2+0:vgprValuB_X2_I0+44+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[188:191] // left value = acc[188+0:191+0] +/* mfmaIndex:240 */ +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[192:195] // left value = acc[192+0:195+0] +/* mfmaIndex:241 */ +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[196:199] // left value = acc[196+0:199+0] +/* mfmaIndex:242 */ +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[200:203] // left value = acc[200+0:203+0] +/* mfmaIndex:243 */ +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X2_I0+48+2+0:vgprValuB_X2_I0+48+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[204:207] // left value = acc[204+0:207+0] +/* mfmaIndex:244 */ +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[208:211] // left value = acc[208+0:211+0] +/* mfmaIndex:245 */ +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[212:215] // left value = acc[212+0:215+0] +/* mfmaIndex:246 */ +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[216:219] // left value = acc[216+0:219+0] +/* mfmaIndex:247 */ +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X2_I0+52+2+0:vgprValuB_X2_I0+52+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[220:223] // left value = acc[220+0:223+0] +/* mfmaIndex:248 */ +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[224:227] // left value = acc[224+0:227+0] +/* mfmaIndex:249 */ +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[228:231] // left value = acc[228+0:231+0] +/* mfmaIndex:250 */ +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[232:235] // left value = acc[232+0:235+0] +/* mfmaIndex:251 */ +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X2_I0+56+2+0:vgprValuB_X2_I0+56+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[236:239] // left value = acc[236+0:239+0] +/* mfmaIndex:252 */ +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+0+2+0:vgprValuA_X2_I0+0+2+0+1], acc[240:243] // left value = acc[240+0:243+0] +/* mfmaIndex:253 */ +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+4+2+0:vgprValuA_X2_I0+4+2+0+1], acc[244:247] // left value = acc[244+0:247+0] +/* mfmaIndex:254 */ +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+8+2+0:vgprValuA_X2_I0+8+2+0+1], acc[248:251] // left value = acc[248+0:251+0] +/* mfmaIndex:255 */ +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X2_I0+60+2+0:vgprValuB_X2_I0+60+2+0+1], v[vgprValuA_X2_I0+12+2+0:vgprValuA_X2_I0+12+2+0+1], acc[252:255] // left value = acc[252+0:255+0] +/* numPrefetchIter=1 */ +/* dataAtIterA=0 numReadsIterA=1 skipReadsIterA=1 readsPerIterA=4 */ +/* dataAtIterB=0 numReadsIterB=1 skipReadsIterB=1 readsPerIterB=16 */ + +s_branch label_Summation_End_OptNLL + +label_PrefetchGlobalLastIterEnd: +.set vgprValuA_X0_I0, vgprValuA_X0_I0_1 +.set vgprValuA_X2_I0, vgprValuA_X2_I0_1 + +/******************************************/ +/* Tail Loop */ +/******************************************/ + +/* Tail: add ValuA/B vgpr buffer [0...160) to pool */ + +/* local write reset offsets a */ + +/* local write reset offsets b */ + +// numIterL = LOCAL_SPLITU * min(sizeL % LOCAL_DEPTHU, DEPTHU / LOCAL_SPLITU) +s_and_b32 s[sgprLoopCounterL], 63, s[sgprSizesSum+0] // s[sgprLoopCounterL] = s[sgprSizesSum+0] % 64 +s_cmp_lg_u32 s[sgprGSUSumIdx], s[sgprGSUSumIdx+1] // gsuSumIdx == numIterPerWgRemainder +s_cmov_b32 s[sgprLoopCounterL], 0x0 // numIter=0 if gsuSimIdx!=remainder +s_cmp_eq_u32 s[sgprLoopCounterL], 0x0 // numIterL == 0 +s_mov_b32 s[sgprOrigLoopCounter], 0 // repurpose to count each localRead increment +s_cbranch_scc1 label_SkipTailLoopL // skip to end of tail loop b/c numIter==0 + +/* remove stagger offsets for tail loop */ +s_sub_i32 s80, 3, s[sgprStaggerUIter] +s_mul_hi_i32 s81, s80, s[sgprGlobalReadIncsA+0] // start offset S in bytes +s_mul_i32 s80, s80, s[sgprGlobalReadIncsA+0] // start offset S in bytes +s_sub_u32 s80, s80, s[sgprWrapUA] // S - WrapU +s_subb_u32 s81, s81, s[sgprWrapUA+1] // S - WrapU +s_add_u32 s[sgprSrdA+0], s[sgprSrdA+0], s80 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdA+1], s[sgprSrdA+1], s81 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitA+0], s[sgprShadowLimitA+0], s80 // limit -= inc) +s_subb_u32 s[sgprShadowLimitA+1], s[sgprShadowLimitA+1], s81 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitA+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdA+2], s[sgprShadowLimitA+0], BufferLimit // Move shadow to real if we are within 2^32 +s_sub_i32 s80, 3, s[sgprStaggerUIter] +s_mul_hi_i32 s81, s80, s[sgprGlobalReadIncsB+0] // start offset S in bytes +s_mul_i32 s80, s80, s[sgprGlobalReadIncsB+0] // start offset S in bytes +s_sub_u32 s80, s80, s[sgprWrapUB] // S - WrapU +s_subb_u32 s81, s81, s[sgprWrapUB+1] // S - WrapU +s_add_u32 s[sgprSrdB+0], s[sgprSrdB+0], s80 // gra SRD += inc(lower) +s_addc_u32 s[sgprSrdB+1], s[sgprSrdB+1], s81 // gra SRD += inc(upper) +s_sub_u32 s[sgprShadowLimitB+0], s[sgprShadowLimitB+0], s80 // limit -= inc) +s_subb_u32 s[sgprShadowLimitB+1], s[sgprShadowLimitB+1], s81 // limit -= inc) +s_cmp_eq_u32 s[sgprShadowLimitB+1], 0 // are we within 2^32? +s_cselect_b32 s[sgprSrdB+2], s[sgprShadowLimitB+0], BufferLimit // Move shadow to real if we are within 2^32 + +/* Recalc global read offsets */ +v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v0 15, v1 // 1. M offset: mIdx = wtid % MI_M(16) +v_mul_lo_u32 v0, s[sgprStrideA0I], v0 // 1. M offset: mOffset = mIdx * mStride(k) +v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) +v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v1, 0x2, v1 // 5. K offset: lrKOffset = kIdx * mStride(4) +v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in M dimen: wtid = tid / dividedForWaveId(64) +v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid % num1DWaves(4) +v_mul_lo_u32 v1, s[sgprStrideA0I], v1 // 7. wave offset in M dimen: wOffset = wtid0 * s[sgprStrideA0I](8192) +v_lshlrev_b32 v1, 0x6, v1 // 7. wave offset in M dimen: wOffset = wOffset * 16 * vw(4) +v_add_u32 v[vgprGlobalReadOffsetA], v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset +v_add_u32 v[vgprGlobalReadOffsetA] 0x8 v[vgprGlobalReadOffsetA] // add prepad for pointer shift +v_lshlrev_b32 v[vgprGlobalReadOffsetA] 0x1 v[vgprGlobalReadOffsetA] // offset *= bytes/element + +s_mul_i32 s[sgprScalarGlobalReadOffsetA+0], s[sgprStrideA0I], 1 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+0], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetA+1], s[sgprStrideA0I], 2 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+1], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetA+2], s[sgprStrideA0I], 3 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+2], 0x1 // scalar offset *= bytes/element +s_mul_i32 s[sgprScalarGlobalReadOffsetA+3], 1, 16 // compute offset diff (scaled tileDim) +s_lshl_b32 s[sgprScalarGlobalReadOffsetA+3], s[sgprScalarGlobalReadOffsetA+3], 0x1 // scalar offset *= bytes/element +s_add_u32 s[sgprScalarGlobalReadOffsetA+4], s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+3] +s_add_u32 s[sgprScalarGlobalReadOffsetA+5], s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+3] +s_add_u32 s[sgprScalarGlobalReadOffsetA+6], s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+3] + +/* Update M0 for DTLDS */ + +/* g2l=0, load component 0 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+0], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:0 // load one buffer value +/* g2l=0, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+0], v[vgprValuA_X0_I0+0], v0 // HasEccHalf: pack +/* g2l=0, load component 2 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+1], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:4 // load one buffer value +/* g2l=0, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+1], v[vgprValuA_X0_I0+1], v0 // HasEccHalf: pack +/* g2l=0, load component 0 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+8], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // load one buffer value +/* g2l=0, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+8], v[vgprValuA_X0_I0+8], v0 // HasEccHalf: pack +/* g2l=0, load component 2 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+9], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:4 // load one buffer value +/* g2l=0, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+9], v[vgprValuA_X0_I0+9], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+3], s[sgprScalarGlobalReadOffsetA+3], 64 +/* g2l=0, load component 0 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+0], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:64 // load one buffer value +/* g2l=0, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:66 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+0], v[vgprValuA_X2_I0+0], v0 // HasEccHalf: pack +/* g2l=0, load component 2 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+1], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:68 // load one buffer value +/* g2l=0, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], 0 offen offset:70 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+1], v[vgprValuA_X2_I0+1], v0 // HasEccHalf: pack +/* g2l=0, load component 0 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+8], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:0 // load one buffer value +/* g2l=0, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+8], v[vgprValuA_X2_I0+8], v0 // HasEccHalf: pack +/* g2l=0, load component 2 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+9], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:4 // load one buffer value +/* g2l=0, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+3] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+9], v[vgprValuA_X2_I0+9], v0 // HasEccHalf: pack +/* g2l=0, load component 4 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+2], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // load one buffer value +/* g2l=0, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+2], v[vgprValuA_X0_I0+2], v0 // HasEccHalf: pack +/* g2l=0, load component 6 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:4 // load one buffer value +/* g2l=0, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+3], v[vgprValuA_X0_I0+3], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+0], s[sgprScalarGlobalReadOffsetA+0], 64 +/* g2l=0, load component 4 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+10], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // load one buffer value +/* g2l=0, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+10], v[vgprValuA_X0_I0+10], v0 // HasEccHalf: pack +/* g2l=0, load component 6 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+11], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:4 // load one buffer value +/* g2l=0, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+11], v[vgprValuA_X0_I0+11], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+4], s[sgprScalarGlobalReadOffsetA+4], 64 +/* g2l=0, load component 4 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+2], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:0 // load one buffer value +/* g2l=0, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+2], v[vgprValuA_X2_I0+2], v0 // HasEccHalf: pack +/* g2l=0, load component 6 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+3], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:4 // load one buffer value +/* g2l=0, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+0] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+3], v[vgprValuA_X2_I0+3], v0 // HasEccHalf: pack +/* g2l=0, load component 4 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+10], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:0 // load one buffer value +/* g2l=0, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+10], v[vgprValuA_X2_I0+10], v0 // HasEccHalf: pack +/* g2l=0, load component 6 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+11], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:4 // load one buffer value +/* g2l=0, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+4] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+11], v[vgprValuA_X2_I0+11], v0 // HasEccHalf: pack +/* g2l=0, load component 8 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+4], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // load one buffer value +/* g2l=0, load component 9 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+4], v[vgprValuA_X0_I0+4], v0 // HasEccHalf: pack +/* g2l=0, load component 10 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+5], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:4 // load one buffer value +/* g2l=0, load component 11 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+5], v[vgprValuA_X0_I0+5], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+1], s[sgprScalarGlobalReadOffsetA+1], 64 +/* g2l=0, load component 8 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+12], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // load one buffer value +/* g2l=0, load component 9 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+12], v[vgprValuA_X0_I0+12], v0 // HasEccHalf: pack +/* g2l=0, load component 10 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+13], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:4 // load one buffer value +/* g2l=0, load component 11 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+13], v[vgprValuA_X0_I0+13], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+5], s[sgprScalarGlobalReadOffsetA+5], 64 +/* g2l=0, load component 8 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+4], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:0 // load one buffer value +/* g2l=0, load component 9 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+4], v[vgprValuA_X2_I0+4], v0 // HasEccHalf: pack +/* g2l=0, load component 10 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+5], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:4 // load one buffer value +/* g2l=0, load component 11 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+1] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+5], v[vgprValuA_X2_I0+5], v0 // HasEccHalf: pack +/* g2l=0, load component 8 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+12], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:0 // load one buffer value +/* g2l=0, load component 9 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+12], v[vgprValuA_X2_I0+12], v0 // HasEccHalf: pack +/* g2l=0, load component 10 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+13], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:4 // load one buffer value +/* g2l=0, load component 11 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+5] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+13], v[vgprValuA_X2_I0+13], v0 // HasEccHalf: pack +/* g2l=0, load component 12 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+6], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // load one buffer value +/* g2l=0, load component 13 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+6], v[vgprValuA_X0_I0+6], v0 // HasEccHalf: pack +/* g2l=0, load component 14 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+7], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:4 // load one buffer value +/* g2l=0, load component 15 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+7], v[vgprValuA_X0_I0+7], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+2], s[sgprScalarGlobalReadOffsetA+2], 64 +/* g2l=0, load component 12 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+14], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // load one buffer value +/* g2l=0, load component 13 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+14], v[vgprValuA_X0_I0+14], v0 // HasEccHalf: pack +/* g2l=0, load component 14 */ +buffer_load_short_d16 v[vgprValuA_X0_I0+15], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:4 // load one buffer value +/* g2l=0, load component 15 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X0_I0+15], v[vgprValuA_X0_I0+15], v0 // HasEccHalf: pack +s_add_u32 s[sgprScalarGlobalReadOffsetA+6], s[sgprScalarGlobalReadOffsetA+6], 64 +/* g2l=0, load component 12 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+6], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:0 // load one buffer value +/* g2l=0, load component 13 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+6], v[vgprValuA_X2_I0+6], v0 // HasEccHalf: pack +/* g2l=0, load component 14 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+7], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:4 // load one buffer value +/* g2l=0, load component 15 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+2] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+7], v[vgprValuA_X2_I0+7], v0 // HasEccHalf: pack +/* g2l=0, load component 12 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+14], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:0 // load one buffer value +/* g2l=0, load component 13 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+14], v[vgprValuA_X2_I0+14], v0 // HasEccHalf: pack +/* g2l=0, load component 14 */ +buffer_load_short_d16 v[vgprValuA_X2_I0+15], v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:4 // load one buffer value +/* g2l=0, load component 15 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetA+0], s[sgprSrdA:sgprSrdA+3], s[sgprScalarGlobalReadOffsetA+6] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprValuA_X2_I0+15], v[vgprValuA_X2_I0+15], v0 // HasEccHalf: pack + +/* Update M0 for DTLDS */ + +/* global read B */ +/* g2l=0, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+0+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:0 // load one buffer value +/* g2l=0, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+0+0], v[vgprG2LB+0+0], v0 // HasEccHalf: pack +/* g2l=0, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+0+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:4 // load one buffer value +/* g2l=0, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+0+1], v[vgprG2LB+0+1], v0 // HasEccHalf: pack +/* g2l=0, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+0+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:8 // load one buffer value +/* g2l=0, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+0+2], v[vgprG2LB+0+2], v0 // HasEccHalf: pack +/* g2l=0, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+0+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:12 // load one buffer value +/* g2l=0, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], 0 offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+0+3], v[vgprG2LB+0+3], v0 // HasEccHalf: pack +/* g2l=4, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+4+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:0 // load one buffer value +/* g2l=4, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+4+0], v[vgprG2LB+4+0], v0 // HasEccHalf: pack +/* g2l=4, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+4+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:4 // load one buffer value +/* g2l=4, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+4+1], v[vgprG2LB+4+1], v0 // HasEccHalf: pack +/* g2l=4, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+4+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:8 // load one buffer value +/* g2l=4, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+4+2], v[vgprG2LB+4+2], v0 // HasEccHalf: pack +/* g2l=4, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+4+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:12 // load one buffer value +/* g2l=4, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+0] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+4+3], v[vgprG2LB+4+3], v0 // HasEccHalf: pack +/* g2l=8, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+8+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:0 // load one buffer value +/* g2l=8, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+8+0], v[vgprG2LB+8+0], v0 // HasEccHalf: pack +/* g2l=8, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+8+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:4 // load one buffer value +/* g2l=8, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+8+1], v[vgprG2LB+8+1], v0 // HasEccHalf: pack +/* g2l=8, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+8+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:8 // load one buffer value +/* g2l=8, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+8+2], v[vgprG2LB+8+2], v0 // HasEccHalf: pack +/* g2l=8, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+8+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:12 // load one buffer value +/* g2l=8, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+1] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+8+3], v[vgprG2LB+8+3], v0 // HasEccHalf: pack +/* g2l=12, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+12+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:0 // load one buffer value +/* g2l=12, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+12+0], v[vgprG2LB+12+0], v0 // HasEccHalf: pack +/* g2l=12, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+12+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:4 // load one buffer value +/* g2l=12, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+12+1], v[vgprG2LB+12+1], v0 // HasEccHalf: pack +/* g2l=12, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+12+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:8 // load one buffer value +/* g2l=12, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+12+2], v[vgprG2LB+12+2], v0 // HasEccHalf: pack +/* g2l=12, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+12+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:12 // load one buffer value +/* g2l=12, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+2] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+12+3], v[vgprG2LB+12+3], v0 // HasEccHalf: pack +/* g2l=16, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+16+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:0 // load one buffer value +/* g2l=16, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+16+0], v[vgprG2LB+16+0], v0 // HasEccHalf: pack +/* g2l=16, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+16+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:4 // load one buffer value +/* g2l=16, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+16+1], v[vgprG2LB+16+1], v0 // HasEccHalf: pack +/* g2l=16, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+16+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:8 // load one buffer value +/* g2l=16, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+16+2], v[vgprG2LB+16+2], v0 // HasEccHalf: pack +/* g2l=16, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+16+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:12 // load one buffer value +/* g2l=16, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+3] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+16+3], v[vgprG2LB+16+3], v0 // HasEccHalf: pack +/* g2l=20, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+20+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:0 // load one buffer value +/* g2l=20, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+20+0], v[vgprG2LB+20+0], v0 // HasEccHalf: pack +/* g2l=20, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+20+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:4 // load one buffer value +/* g2l=20, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+20+1], v[vgprG2LB+20+1], v0 // HasEccHalf: pack +/* g2l=20, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+20+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:8 // load one buffer value +/* g2l=20, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+20+2], v[vgprG2LB+20+2], v0 // HasEccHalf: pack +/* g2l=20, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+20+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:12 // load one buffer value +/* g2l=20, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+4] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+20+3], v[vgprG2LB+20+3], v0 // HasEccHalf: pack +/* g2l=24, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+24+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:0 // load one buffer value +/* g2l=24, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+24+0], v[vgprG2LB+24+0], v0 // HasEccHalf: pack +/* g2l=24, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+24+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:4 // load one buffer value +/* g2l=24, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+24+1], v[vgprG2LB+24+1], v0 // HasEccHalf: pack +/* g2l=24, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+24+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:8 // load one buffer value +/* g2l=24, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+24+2], v[vgprG2LB+24+2], v0 // HasEccHalf: pack +/* g2l=24, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+24+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:12 // load one buffer value +/* g2l=24, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+5] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+24+3], v[vgprG2LB+24+3], v0 // HasEccHalf: pack +/* g2l=28, load component 0 */ +buffer_load_short_d16 v[vgprG2LB+28+0], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:0 // load one buffer value +/* g2l=28, load component 1 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:2 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+28+0], v[vgprG2LB+28+0], v0 // HasEccHalf: pack +/* g2l=28, load component 2 */ +buffer_load_short_d16 v[vgprG2LB+28+1], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:4 // load one buffer value +/* g2l=28, load component 3 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:6 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+28+1], v[vgprG2LB+28+1], v0 // HasEccHalf: pack +/* g2l=28, load component 4 */ +buffer_load_short_d16 v[vgprG2LB+28+2], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:8 // load one buffer value +/* g2l=28, load component 5 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:10 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+28+2], v[vgprG2LB+28+2], v0 // HasEccHalf: pack +/* g2l=28, load component 6 */ +buffer_load_short_d16 v[vgprG2LB+28+3], v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:12 // load one buffer value +/* g2l=28, load component 7 */ +buffer_load_short_d16_hi v0, v[vgprGlobalReadOffsetB+0], s[sgprSrdB:sgprSrdB+3], s[sgprScalarGlobalReadOffsetB+6] offen offset:14 // load one buffer value +s_waitcnt vmcnt(0) +v_or_b32 v[vgprG2LB+28+3], v[vgprG2LB+28+3], v0 // HasEccHalf: pack +s_waitcnt vmcnt(0) // 2wait for global read +// Skip force waitcnt0 +s_barrier + +/* local write a */ + +/* local write b */ +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+0:vgprG2LB+0+3] offset:0 // lwoB_0_0_0_0 = (0*LSCB)*(MT1J+PAD) + (0*LSPB) = 0 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+4:vgprG2LB+4+3] offset:5120 // lwoB_0_0_1_0 = (0*LSCB)*(MT1J+PAD) + (1*LSPB) = 5120 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+8:vgprG2LB+8+3] offset:10240 // lwoB_0_0_2_0 = (0*LSCB)*(MT1J+PAD) + (2*LSPB) = 10240 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+12:vgprG2LB+12+3] offset:15360 // lwoB_0_0_3_0 = (0*LSCB)*(MT1J+PAD) + (3*LSPB) = 15360 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+16:vgprG2LB+16+3] offset:20480 // lwoB_0_0_4_0 = (0*LSCB)*(MT1J+PAD) + (4*LSPB) = 20480 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+20:vgprG2LB+20+3] offset:25600 // lwoB_0_0_5_0 = (0*LSCB)*(MT1J+PAD) + (5*LSPB) = 25600 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+24:vgprG2LB+24+3] offset:30720 // lwoB_0_0_6_0 = (0*LSCB)*(MT1J+PAD) + (6*LSPB) = 30720 +ds_write_b128 v[vgprLocalWriteAddrB], v[vgprG2LB+28:vgprG2LB+28+3] offset:35840 // lwoB_0_0_7_0 = (0*LSCB)*(MT1J+PAD) + (7*LSPB) = 35840 + +/* Recalc local read offsets */ +/* lr0I */ +v_and_b32 v1, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v0, 15, v1 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v0, 0x6, v0 // 1. N offset: nOffset = nIdx * nStride(64) +/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ +v_lshlrev_b32 v0, 0x2, v0 // 4. apply VectorWidth: bnOffset = bnOffset * vw(4) +v_and_b32 v1, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v1, 4, v1 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v1, 0x2, v1 // 5. K offset: lrKOffset = kIdx * mStride(4) +v_add_u32 v0, v1, v0 // 6. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v1, 6, v[vgprSerial] // 7. wave offset in N dimen: wtid = tid / dividedForWaveId(64) +v_and_b32 v1, 3, v1 // 7. wave offset in M dimen: wtid0 = wtid / num1DWaves(4) +v_lshlrev_b32 v1, 0xc, v1 // 7. wave offset in M dimen: wOffset = wtid0 * W0Stride(4096) +v_add_u32 v0, v1, v0 // 7. final local read offset: flrOffset = lrOffset + WOffset +/* lr1J */ +v_and_b32 v2, 63, v[vgprSerial] // 0. thread id in wave: wtid = tid % wavelength(64) +v_and_b32 v1, 15, v2 // 1. N offset: nIdx = wtid % MI_N(16) +v_lshlrev_b32 v1, 0x6, v1 // 1. N offset: nOffset = nIdx * nStride(64) +/* Skip. 2. block offset: bnOffset = 0 when num1DBlocks = 1 */ + // 4. apply VectorWidth: bnOffset = bnOffset * vw(1) (multiplier is 1, do nothing) +v_and_b32 v2, 63, v[vgprSerial] // 5. thread id in wave: wtid = tid % wavelength(64) +v_lshrrev_b32 v2, 4, v2 // 5. K offset: kIdx = wtid / (MIN(16) * MIBB(1)) +v_lshlrev_b32 v2, 0x2, v2 // 5. K offset: lrKOffset = kIdx * mStride(4) +v_add_u32 v1, v2, v1 // 6. offset in wave: lrOffset = bnOffset + lrKOffset +v_lshrrev_b32 v2, 6, v[vgprSerial] // v2 = v[vgprSerial] / 64 +v_lshrrev_b32 v2, 2, v2 // LSU offset: Get LSU wave_id +s_mov_b32 s8, 64 // LSU offset: stride = lsuStride(64) when umlds==True +v_mul_lo_u32 v2, s8, v2 // LSU offset: lsuoffset = wave_id*lsuStride*(MT0+PAD) +v_add_lshl_u32 v[vgprLocalReadAddrA], v2, v0, 0x1 // Final Offset: offset = (lro0+lsuoffset)*bpeDS +v_lshrrev_b32 v3, 9, v[vgprLocalReadAddrA] // Final Offset: padding 32 per block 512 +v_lshlrev_b32 v3, 0x5, v3 // Final Offset: padding 32 per block 512 +v_add_u32 v[vgprLocalReadAddrA], v3, v[vgprLocalReadAddrA] // Final Offset: add padding 32 per block 512 +/* N/A */ +v_lshrrev_b32 v0, 6, v[vgprSerial] // v0 = v[vgprSerial] / 64 +v_lshrrev_b32 v0, 2, v0 // LSU offset: Get LSU wave_id +s_mov_b32 s8, 64 // LSU offset: stride = lsuStride(64) when umlds==True +v_mul_lo_u32 v0, s8, v0 // LSU offset: lsuoffset = wave_id*lsuStride*(MT1+PAD) +v_add_lshl_u32 v[vgprLocalReadAddrB], v0, v1, 0x1 // Final Offset: offset = (lro1+lsuoffset)*bpeDS +v_lshrrev_b32 v2, 7, v[vgprLocalReadAddrB] // Final Offset: padding 32 per block 128 +v_lshlrev_b32 v2, 0x5, v2 // Final Offset: padding 32 per block 128 +v_add_u32 v[vgprLocalReadAddrB], v2, v[vgprLocalReadAddrB] // Final Offset: add padding 32 per block 128 +s_waitcnt lgkmcnt(0) // 5wait for local write +// Skip force waitcnt0 +s_barrier + +/* local read reset offsets a */ + +/* local read reset offsets b */ + +/* local read init pointers a */ + +/* localReadInitPointers */ + +/* local read init pointers b */ + +/* localReadInitPointers */ + +/* tail loop: macs */ +label_TailLoopBeginL: + +/* Tail: remove ValuA/B vgpr buffer [0...160) from pool */ + +/* Tail: add address/G2L vgpr [160...230) to pool */ + +/* local read a */ + +/* local read b */ +ds_read_b64 v[vgprValuB_X0_I0+0:vgprValuB_X0_I0+0+1], v[vgprLocalReadAddrB] offset:0 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=0 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+2:vgprValuB_X0_I0+2+1], v[vgprLocalReadAddrB] offset:2560 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=1 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+4:vgprValuB_X0_I0+4+1], v[vgprLocalReadAddrB] offset:5120 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=2 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+6:vgprValuB_X0_I0+6+1], v[vgprLocalReadAddrB] offset:7680 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=3 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+8:vgprValuB_X0_I0+8+1], v[vgprLocalReadAddrB] offset:10240 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=4 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+10:vgprValuB_X0_I0+10+1], v[vgprLocalReadAddrB] offset:12800 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=5 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+12:vgprValuB_X0_I0+12+1], v[vgprLocalReadAddrB] offset:15360 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=6 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+14:vgprValuB_X0_I0+14+1], v[vgprLocalReadAddrB] offset:17920 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=7 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+16:vgprValuB_X0_I0+16+1], v[vgprLocalReadAddrB] offset:20480 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=8 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+18:vgprValuB_X0_I0+18+1], v[vgprLocalReadAddrB] offset:23040 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=9 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+20:vgprValuB_X0_I0+20+1], v[vgprLocalReadAddrB] offset:25600 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=10 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+22:vgprValuB_X0_I0+22+1], v[vgprLocalReadAddrB] offset:28160 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=11 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+24:vgprValuB_X0_I0+24+1], v[vgprLocalReadAddrB] offset:30720 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=12 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+26:vgprValuB_X0_I0+26+1], v[vgprLocalReadAddrB] offset:33280 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=13 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+28:vgprValuB_X0_I0+28+1], v[vgprLocalReadAddrB] offset:35840 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=14 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 +ds_read_b64 v[vgprValuB_X0_I0+30:vgprValuB_X0_I0+30+1], v[vgprLocalReadAddrB] offset:38400 // L -> Reg lro=0 swapByteOffset=0 ti=16 vIdx=15 eIdx=0 rIdx=0 oIdx=0 buffer=0 iui=0 + +/* local read inc a */ +s_mov_b32 s8, 0x20 // inc + +/* local read inc b */ +s_mov_b32 s8, 0x20 // inc +v_add_co_u32 v[vgprLocalReadAddrB], vcc, s8, v[vgprLocalReadAddrB] // lrB += 32 (bpeDS) +s_waitcnt lgkmcnt(0) // 4wait for local read +v_and_b32 v160, 63, v[vgprSerial] // v160 = v[vgprSerial] % 64 +v_lshrrev_b32 v160, 4, v160 // v160 = v160 / 16 +v_lshlrev_b32 v160, 0x2, v160 // v160 = v160 * 4 +v_cmp_ge_i32 s[80:81], v160, s[sgprLoopCounterL] // check K index >= Size L +v_cndmask_b32 v[vgprValuA_X0_I0+0+0], v[vgprValuA_X0_I0+0+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+2+0], v[vgprValuA_X0_I0+2+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+4+0], v[vgprValuA_X0_I0+4+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+6+0], v[vgprValuA_X0_I0+6+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+0+1], v[vgprValuA_X0_I0+0+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+2+1], v[vgprValuA_X0_I0+2+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+4+1], v[vgprValuA_X0_I0+4+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuA_X0_I0+6+1], v[vgprValuA_X0_I0+6+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+0+0], v[vgprValuB_X0_I0+0+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+2+0], v[vgprValuB_X0_I0+2+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+4+0], v[vgprValuB_X0_I0+4+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+6+0], v[vgprValuB_X0_I0+6+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+8+0], v[vgprValuB_X0_I0+8+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+10+0], v[vgprValuB_X0_I0+10+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+12+0], v[vgprValuB_X0_I0+12+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+14+0], v[vgprValuB_X0_I0+14+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+16+0], v[vgprValuB_X0_I0+16+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+18+0], v[vgprValuB_X0_I0+18+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+20+0], v[vgprValuB_X0_I0+20+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+22+0], v[vgprValuB_X0_I0+22+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+24+0], v[vgprValuB_X0_I0+24+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+26+0], v[vgprValuB_X0_I0+26+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+28+0], v[vgprValuB_X0_I0+28+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+30+0], v[vgprValuB_X0_I0+30+0], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+0+1], v[vgprValuB_X0_I0+0+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+2+1], v[vgprValuB_X0_I0+2+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+4+1], v[vgprValuB_X0_I0+4+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+6+1], v[vgprValuB_X0_I0+6+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+8+1], v[vgprValuB_X0_I0+8+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+10+1], v[vgprValuB_X0_I0+10+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+12+1], v[vgprValuB_X0_I0+12+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+14+1], v[vgprValuB_X0_I0+14+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+16+1], v[vgprValuB_X0_I0+16+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+18+1], v[vgprValuB_X0_I0+18+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+20+1], v[vgprValuB_X0_I0+20+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+22+1], v[vgprValuB_X0_I0+22+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+24+1], v[vgprValuB_X0_I0+24+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+26+1], v[vgprValuB_X0_I0+26+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+28+1], v[vgprValuB_X0_I0+28+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_cndmask_b32 v[vgprValuB_X0_I0+30+1], v[vgprValuB_X0_I0+30+1], 0x0, s[80:81] // set 0 if K_idx >= sizeL +v_sub_u32 v160, s[sgprLoopCounterL], v160 // get distance between size and k index +v_cmp_lt_i32 s[80:81], v160, 4 // set partial 0 if distance less than input per thread +s_and_b32 s82, s[sgprLoopCounterL], 3 // get inputs for edge thread +s_sub_u32 s82, 4, s82 // use shift to fill 0 for outside element +s_lshl_b32 s82, s82, 4 // use shift to fill 0 for outside element +v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1] +v_cndmask_b32 v[vgprValuA_X0_I0+0+0+0+0], v[vgprValuA_X0_I0+0+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuA_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1] +v_cndmask_b32 v[vgprValuA_X0_I0+2+0+0+0], v[vgprValuA_X0_I0+2+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuA_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+2+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1] +v_cndmask_b32 v[vgprValuA_X0_I0+4+0+0+0], v[vgprValuA_X0_I0+4+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuA_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1] +v_cndmask_b32 v[vgprValuA_X0_I0+6+0+0+0], v[vgprValuA_X0_I0+6+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuA_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+6+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+0+0+0+0], v[vgprValuB_X0_I0+0+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+0+0+0+1], v[vgprValuB_X0_I0+0+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+2+0+0+0], v[vgprValuB_X0_I0+2+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+2+0+0+1], v[vgprValuB_X0_I0+2+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+4+0+0+0], v[vgprValuB_X0_I0+4+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+4+0+0+1], v[vgprValuB_X0_I0+4+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+6+0+0+0], v[vgprValuB_X0_I0+6+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+6+0+0+1], v[vgprValuB_X0_I0+6+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+8+0+0+0], v[vgprValuB_X0_I0+8+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+8+0+0+1], v[vgprValuB_X0_I0+8+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+10+0+0+0], v[vgprValuB_X0_I0+10+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+10+0+0+1], v[vgprValuB_X0_I0+10+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+12+0+0+0], v[vgprValuB_X0_I0+12+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+12+0+0+1], v[vgprValuB_X0_I0+12+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+14+0+0+0], v[vgprValuB_X0_I0+14+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+14+0+0+1], v[vgprValuB_X0_I0+14+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+16+0+0+0], v[vgprValuB_X0_I0+16+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+16+0+0+1], v[vgprValuB_X0_I0+16+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+18+0+0+0], v[vgprValuB_X0_I0+18+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+18+0+0+1], v[vgprValuB_X0_I0+18+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+20+0+0+0], v[vgprValuB_X0_I0+20+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+20+0+0+1], v[vgprValuB_X0_I0+20+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+22+0+0+0], v[vgprValuB_X0_I0+22+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+22+0+0+1], v[vgprValuB_X0_I0+22+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+24+0+0+0], v[vgprValuB_X0_I0+24+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+24+0+0+1], v[vgprValuB_X0_I0+24+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+26+0+0+0], v[vgprValuB_X0_I0+26+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+26+0+0+1], v[vgprValuB_X0_I0+26+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+28+0+0+0], v[vgprValuB_X0_I0+28+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+28+0+0+1], v[vgprValuB_X0_I0+28+0+0+1], v163, s[80:81] +v_lshlrev_b64 v[162:163], s82, v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1] +v_cndmask_b32 v[vgprValuB_X0_I0+30+0+0+0], v[vgprValuB_X0_I0+30+0+0+0], v162, s[80:81] +v_cndmask_b32 v[vgprValuB_X0_I0+30+0+0+1], v[vgprValuB_X0_I0+30+0+0+1], v163, s[80:81] +s_nop 1 +v_mfma_f32_16x16x16bf16_1k acc[0:3], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[0:3] // left value = acc[0+0:3+0] +v_mfma_f32_16x16x16bf16_1k acc[4:7], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[4:7] // left value = acc[4+0:7+0] +v_mfma_f32_16x16x16bf16_1k acc[8:11], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[8:11] // left value = acc[8+0:11+0] +v_mfma_f32_16x16x16bf16_1k acc[12:15], v[vgprValuB_X0_I0+0+0+0:vgprValuB_X0_I0+0+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[12:15] // left value = acc[12+0:15+0] +v_mfma_f32_16x16x16bf16_1k acc[16:19], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[16:19] // left value = acc[16+0:19+0] +v_mfma_f32_16x16x16bf16_1k acc[20:23], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[20:23] // left value = acc[20+0:23+0] +v_mfma_f32_16x16x16bf16_1k acc[24:27], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[24:27] // left value = acc[24+0:27+0] +v_mfma_f32_16x16x16bf16_1k acc[28:31], v[vgprValuB_X0_I0+2+0+0:vgprValuB_X0_I0+2+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[28:31] // left value = acc[28+0:31+0] +v_mfma_f32_16x16x16bf16_1k acc[32:35], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[32:35] // left value = acc[32+0:35+0] +v_mfma_f32_16x16x16bf16_1k acc[36:39], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[36:39] // left value = acc[36+0:39+0] +v_mfma_f32_16x16x16bf16_1k acc[40:43], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[40:43] // left value = acc[40+0:43+0] +v_mfma_f32_16x16x16bf16_1k acc[44:47], v[vgprValuB_X0_I0+4+0+0:vgprValuB_X0_I0+4+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[44:47] // left value = acc[44+0:47+0] +v_mfma_f32_16x16x16bf16_1k acc[48:51], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[48:51] // left value = acc[48+0:51+0] +v_mfma_f32_16x16x16bf16_1k acc[52:55], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[52:55] // left value = acc[52+0:55+0] +v_mfma_f32_16x16x16bf16_1k acc[56:59], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[56:59] // left value = acc[56+0:59+0] +v_mfma_f32_16x16x16bf16_1k acc[60:63], v[vgprValuB_X0_I0+6+0+0:vgprValuB_X0_I0+6+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[60:63] // left value = acc[60+0:63+0] +v_mfma_f32_16x16x16bf16_1k acc[64:67], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[64:67] // left value = acc[64+0:67+0] +v_mfma_f32_16x16x16bf16_1k acc[68:71], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[68:71] // left value = acc[68+0:71+0] +v_mfma_f32_16x16x16bf16_1k acc[72:75], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[72:75] // left value = acc[72+0:75+0] +v_mfma_f32_16x16x16bf16_1k acc[76:79], v[vgprValuB_X0_I0+8+0+0:vgprValuB_X0_I0+8+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[76:79] // left value = acc[76+0:79+0] +v_mfma_f32_16x16x16bf16_1k acc[80:83], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[80:83] // left value = acc[80+0:83+0] +v_mfma_f32_16x16x16bf16_1k acc[84:87], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[84:87] // left value = acc[84+0:87+0] +v_mfma_f32_16x16x16bf16_1k acc[88:91], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[88:91] // left value = acc[88+0:91+0] +v_mfma_f32_16x16x16bf16_1k acc[92:95], v[vgprValuB_X0_I0+10+0+0:vgprValuB_X0_I0+10+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[92:95] // left value = acc[92+0:95+0] +v_mfma_f32_16x16x16bf16_1k acc[96:99], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[96:99] // left value = acc[96+0:99+0] +v_mfma_f32_16x16x16bf16_1k acc[100:103], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[100:103] // left value = acc[100+0:103+0] +v_mfma_f32_16x16x16bf16_1k acc[104:107], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[104:107] // left value = acc[104+0:107+0] +v_mfma_f32_16x16x16bf16_1k acc[108:111], v[vgprValuB_X0_I0+12+0+0:vgprValuB_X0_I0+12+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[108:111] // left value = acc[108+0:111+0] +v_mfma_f32_16x16x16bf16_1k acc[112:115], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[112:115] // left value = acc[112+0:115+0] +v_mfma_f32_16x16x16bf16_1k acc[116:119], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[116:119] // left value = acc[116+0:119+0] +v_mfma_f32_16x16x16bf16_1k acc[120:123], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[120:123] // left value = acc[120+0:123+0] +v_mfma_f32_16x16x16bf16_1k acc[124:127], v[vgprValuB_X0_I0+14+0+0:vgprValuB_X0_I0+14+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[124:127] // left value = acc[124+0:127+0] +v_mfma_f32_16x16x16bf16_1k acc[128:131], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[128:131] // left value = acc[128+0:131+0] +v_mfma_f32_16x16x16bf16_1k acc[132:135], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[132:135] // left value = acc[132+0:135+0] +v_mfma_f32_16x16x16bf16_1k acc[136:139], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[136:139] // left value = acc[136+0:139+0] +v_mfma_f32_16x16x16bf16_1k acc[140:143], v[vgprValuB_X0_I0+16+0+0:vgprValuB_X0_I0+16+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[140:143] // left value = acc[140+0:143+0] +v_mfma_f32_16x16x16bf16_1k acc[144:147], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[144:147] // left value = acc[144+0:147+0] +v_mfma_f32_16x16x16bf16_1k acc[148:151], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[148:151] // left value = acc[148+0:151+0] +v_mfma_f32_16x16x16bf16_1k acc[152:155], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[152:155] // left value = acc[152+0:155+0] +v_mfma_f32_16x16x16bf16_1k acc[156:159], v[vgprValuB_X0_I0+18+0+0:vgprValuB_X0_I0+18+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[156:159] // left value = acc[156+0:159+0] +v_mfma_f32_16x16x16bf16_1k acc[160:163], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[160:163] // left value = acc[160+0:163+0] +v_mfma_f32_16x16x16bf16_1k acc[164:167], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[164:167] // left value = acc[164+0:167+0] +v_mfma_f32_16x16x16bf16_1k acc[168:171], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[168:171] // left value = acc[168+0:171+0] +v_mfma_f32_16x16x16bf16_1k acc[172:175], v[vgprValuB_X0_I0+20+0+0:vgprValuB_X0_I0+20+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[172:175] // left value = acc[172+0:175+0] +v_mfma_f32_16x16x16bf16_1k acc[176:179], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[176:179] // left value = acc[176+0:179+0] +v_mfma_f32_16x16x16bf16_1k acc[180:183], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[180:183] // left value = acc[180+0:183+0] +v_mfma_f32_16x16x16bf16_1k acc[184:187], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[184:187] // left value = acc[184+0:187+0] +v_mfma_f32_16x16x16bf16_1k acc[188:191], v[vgprValuB_X0_I0+22+0+0:vgprValuB_X0_I0+22+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[188:191] // left value = acc[188+0:191+0] +v_mfma_f32_16x16x16bf16_1k acc[192:195], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[192:195] // left value = acc[192+0:195+0] +v_mfma_f32_16x16x16bf16_1k acc[196:199], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[196:199] // left value = acc[196+0:199+0] +v_mfma_f32_16x16x16bf16_1k acc[200:203], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[200:203] // left value = acc[200+0:203+0] +v_mfma_f32_16x16x16bf16_1k acc[204:207], v[vgprValuB_X0_I0+24+0+0:vgprValuB_X0_I0+24+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[204:207] // left value = acc[204+0:207+0] +v_mfma_f32_16x16x16bf16_1k acc[208:211], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[208:211] // left value = acc[208+0:211+0] +v_mfma_f32_16x16x16bf16_1k acc[212:215], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[212:215] // left value = acc[212+0:215+0] +v_mfma_f32_16x16x16bf16_1k acc[216:219], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[216:219] // left value = acc[216+0:219+0] +v_mfma_f32_16x16x16bf16_1k acc[220:223], v[vgprValuB_X0_I0+26+0+0:vgprValuB_X0_I0+26+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[220:223] // left value = acc[220+0:223+0] +v_mfma_f32_16x16x16bf16_1k acc[224:227], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[224:227] // left value = acc[224+0:227+0] +v_mfma_f32_16x16x16bf16_1k acc[228:231], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[228:231] // left value = acc[228+0:231+0] +v_mfma_f32_16x16x16bf16_1k acc[232:235], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[232:235] // left value = acc[232+0:235+0] +v_mfma_f32_16x16x16bf16_1k acc[236:239], v[vgprValuB_X0_I0+28+0+0:vgprValuB_X0_I0+28+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[236:239] // left value = acc[236+0:239+0] +v_mfma_f32_16x16x16bf16_1k acc[240:243], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+0+0+0:vgprValuA_X0_I0+0+0+0+1], acc[240:243] // left value = acc[240+0:243+0] +v_mfma_f32_16x16x16bf16_1k acc[244:247], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+2+0+0:vgprValuA_X0_I0+2+0+0+1], acc[244:247] // left value = acc[244+0:247+0] +v_mfma_f32_16x16x16bf16_1k acc[248:251], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+4+0+0:vgprValuA_X0_I0+4+0+0+1], acc[248:251] // left value = acc[248+0:251+0] +v_mfma_f32_16x16x16bf16_1k acc[252:255], v[vgprValuB_X0_I0+30+0+0:vgprValuB_X0_I0+30+0+0+1], v[vgprValuA_X0_I0+6+0+0:vgprValuA_X0_I0+6+0+0+1], acc[252:255] // left value = acc[252+0:255+0] + +v_mov_b32 v[vgprValuA_X0_I0+0+0], v[vgprValuA_X0_I0+8+0] +v_mov_b32 v[vgprValuA_X0_I0+1+0], v[vgprValuA_X0_I0+9+0] +v_mov_b32 v[vgprValuA_X0_I0+2+0], v[vgprValuA_X0_I0+10+0] +v_mov_b32 v[vgprValuA_X0_I0+3+0], v[vgprValuA_X0_I0+11+0] +v_mov_b32 v[vgprValuA_X0_I0+4+0], v[vgprValuA_X0_I0+12+0] +v_mov_b32 v[vgprValuA_X0_I0+5+0], v[vgprValuA_X0_I0+13+0] +v_mov_b32 v[vgprValuA_X0_I0+6+0], v[vgprValuA_X0_I0+14+0] +v_mov_b32 v[vgprValuA_X0_I0+7+0], v[vgprValuA_X0_I0+15+0] +v_mov_b32 v[vgprValuA_X0_I0+8+0], v[vgprValuA_X2_I0+0+0] +v_mov_b32 v[vgprValuA_X0_I0+9+0], v[vgprValuA_X2_I0+1+0] +v_mov_b32 v[vgprValuA_X0_I0+10+0], v[vgprValuA_X2_I0+2+0] +v_mov_b32 v[vgprValuA_X0_I0+11+0], v[vgprValuA_X2_I0+3+0] +v_mov_b32 v[vgprValuA_X0_I0+12+0], v[vgprValuA_X2_I0+4+0] +v_mov_b32 v[vgprValuA_X0_I0+13+0], v[vgprValuA_X2_I0+5+0] +v_mov_b32 v[vgprValuA_X0_I0+14+0], v[vgprValuA_X2_I0+6+0] +v_mov_b32 v[vgprValuA_X0_I0+15+0], v[vgprValuA_X2_I0+7+0] +v_mov_b32 v[vgprValuA_X2_I0+0+0], v[vgprValuA_X2_I0+8+0] +v_mov_b32 v[vgprValuA_X2_I0+1+0], v[vgprValuA_X2_I0+9+0] +v_mov_b32 v[vgprValuA_X2_I0+2+0], v[vgprValuA_X2_I0+10+0] +v_mov_b32 v[vgprValuA_X2_I0+3+0], v[vgprValuA_X2_I0+11+0] +v_mov_b32 v[vgprValuA_X2_I0+4+0], v[vgprValuA_X2_I0+12+0] +v_mov_b32 v[vgprValuA_X2_I0+5+0], v[vgprValuA_X2_I0+13+0] +v_mov_b32 v[vgprValuA_X2_I0+6+0], v[vgprValuA_X2_I0+14+0] +v_mov_b32 v[vgprValuA_X2_I0+7+0], v[vgprValuA_X2_I0+15+0] + +/* closeLoop loopL finalLoop=1 tailLoop=1 */ +s_sub_i32 s[sgprLoopCounterL], s[sgprLoopCounterL], 0x10 // dec counterL (tailLoop) +s_add_u32 s[sgprOrigLoopCounter], s[sgprOrigLoopCounter], 0x10 // inc counterL +s_cmp_le_i32 s[sgprLoopCounterL], 0x0 // counterL<=0 +s_cbranch_scc0 label_TailLoopBeginL // restart LoopL +label_TailLoopEndL: +label_SkipTailLoopL: + +/* Tail: remove address/G2L [160...230) from pool */ +label_Summation_End_SB904UR36QKZ73J2_0: +/* endSummation: add vgpr [0...230) to pool */ +.set sgprWGM, UNDEF +.set sgprLoopCounterL, UNDEF +.set sgprOrigLoopCounter, UNDEF +.set sgprAddressA, UNDEF +.set sgprAddressB, UNDEF +.set sgprStridesA, UNDEF +.set sgprStridesB, UNDEF +.set sgprStaggerUIter, UNDEF +.set sgprSrdA, UNDEF +.set sgprSrdB, UNDEF +.set sgprShadowLimitA, UNDEF +.set sgprShadowLimitB, UNDEF +.set sgprWrapUA, UNDEF +.set sgprWrapUB, UNDEF +.set sgprGlobalReadIncsA, UNDEF +.set sgprGlobalReadIncsB, UNDEF +.set sgprScalarGlobalReadOffsetA, UNDEF +.set sgprScalarGlobalReadOffsetB, UNDEF +/* load store sgprs */ +.set sgprAddressScaleAlphaVec, 48 +.set sgprAddressBias, 50 +.set sgprBiasType, 52 +.set sgprBiasStride, 53 +.set sgpractivationAlpha, 54 +.set sgpractivationBeta, 55 +.set sgprActivationType, 56 +s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? +s_cbranch_scc0 label_GSU_4 // branch if GSU != 1 +/* Check if custom structure pointer is null */ +s_cmp_eq_u32 s[sgprArgType], 2 // ArgType == 2 ? +s_cbranch_scc1 label_LoadExternalEpilogueStruct_1 // branch if ArgType == 2 +s_load_dwordx8 s[48:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x58 +s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0x78 +s_branch label_LoadExternalEpilogueStructEnd_1 +label_LoadExternalEpilogueStruct_1: +s_load_dwordx4 s[48:51], s[sgprKernArgAddress:sgprKernArgAddress+1], 0x90 +s_load_dwordx2 s[52:53], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xa0 +s_load_dwordx2 s[54:55], s[sgprKernArgAddress:sgprKernArgAddress+1], 0xb8 +s_load_dword s56, s[sgprKernArgAddress:sgprKernArgAddress+1], 0xc0 +label_LoadExternalEpilogueStructEnd_1: +label_GSU_4: +.set sgprSrdScaleAlphaVec, 32 +.set sgprSrdBias, 40 + +/* Mapping of Acc register -> C Vgpr register */ + +/* not-LocalSplitU: global write indices */ +/* computeStoreVgprs */ +v_lshrrev_b32 v4, 6, v[vgprSerial] // v4 = v[vgprSerial] / 64 +v_lshrrev_b32 v5, 2, v4 // v5 = v4 / 4 +v_mul_lo_u32 v5, 0x10, v5 // wave coordination offset 1 +v_and_b32 v1, 63, v[vgprSerial] // v1 = v[vgprSerial] % 64 +v_lshrrev_b32 v1, 4, v1 // v1 = v1 / 16 +v_lshlrev_b32 v1, 0x2, v1 // thread0 * continuous_output +v_add_lshl_u32 v1, v5, v1, 0 // coordination 1 = vwB *(wave_id1 + tid1) +v_mul_lo_u32 v2, v1, s[sgprStrideC1J] // offset 1 +v_mul_lo_u32 v3, v1, s[sgprStrideD1J] // offset 1 +v_and_b32 v0, 3, v4 // v0 = v4 % 4 +v_mul_lo_u32 v0, 0x10, v0 // wave coordination offset 0 +v_and_b32 v5, 15, v[vgprSerial] // v5 = v[vgprSerial] % 16 +v_add_lshl_u32 v0, v5, v0, 2 // coordination 0 = vwA * (wave_id0 + tid0) +s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v0, s8, v0 // coord 0 = (tid0/MI_m)*4 + waveG0*MIB_m + MT0*SG0 +s_mul_i32 s8, 256, s[sgprWorkGroup1] // wgp1 * MT1 +v_add_u32 v1, s8, v1 // coord 1 = (tid0%MI_m) + waveG1*MIB_n + MT1*SG1 + +/* not-LocalSplitU: global write */ + +/******************************************/ +/* Global Write Elements */ +/******************************************/ +s_waitcnt lgkmcnt(0) // wait for 36 bytes of kern args. +s_cmp_eq_u32 s[sgprGSU], 1 // GSU == 1 ? +s_cbranch_scc1 label_GSU_5 // branch if GSU == 1 +s_and_b32 s58, 255, s[sgprSizeI] // s58 = s[sgprSizeI] % 256 +s_add_u32 s59, -0x1, s[sgprNumWorkGroups0] +s_cmp_ge_u32 s[sgprWorkGroup0], s59 // wg0 >= nwg0-1 ? +s_cselect_b32 s58, s58, 0 // set rMT0 +s_cmpk_gt_u32 s58, 0x0 // rMT0 > 0 +s_cbranch_scc1 label_GW_B0_E1_M // jump if edges required +s_and_b32 s58, 255, s[sgprSizeJ] // s58 = s[sgprSizeJ] % 256 +s_add_u32 s59, -0x1, s[sgprNumWorkGroups1] +s_cmp_ge_u32 s[sgprWorkGroup1], s59 // wg1 >= nwg1-1 +s_cselect_b32 s58, s58, 0 // set rMT1 +s_cmpk_gt_u32 s58, 0x0 // rMT1 > 0 +s_cbranch_scc1 label_GW_B0_E1_N // jump if edges required +label_GW_B0_E0_1: + +/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_lshl_u32 v10, v3, v0, 0x2 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 +v_accvgpr_read_b32 v[vgprValuC+12], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+13], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+14], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+15], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+16], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+17], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+18], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+19], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+20], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+21], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+22], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+23], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+24], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+25], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+26], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+27], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+28], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+29], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+30], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+31], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+32], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+33], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+34], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+35], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+36], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+37], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+38], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+39], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+40], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+41], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+42], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+43], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+44], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+45], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+46], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+47], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+48], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+49], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+50], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+51], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+52], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+53], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+54], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+55], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+56], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+57], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+58], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+59], acc47 // copy acc to vreg[47] +v_accvgpr_read_b32 v[vgprValuC+60], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+61], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+62], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+63], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+64], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+65], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+66], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+67], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+68], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+69], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+70], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+71], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+72], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+73], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+74], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+75], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+12], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+13], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+14], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+15], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+16], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+17], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+18], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+19], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+20], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+21], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+22], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+23], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+24], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+25], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+26], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+27], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+28], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+29], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+30], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+31], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+32], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+33], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+34], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+35], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+36], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+37], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+38], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+39], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+40], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+41], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+42], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+43], acc95 // copy acc to vreg[95] +v_accvgpr_read_b32 v[vgprValuC+44], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+45], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+46], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+47], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+48], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+49], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+50], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+51], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+52], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+53], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+54], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+55], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+56], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+57], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+58], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+59], acc111 // copy acc to vreg[111] +v_accvgpr_read_b32 v[vgprValuC+60], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+61], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+62], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+63], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+64], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+65], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+66], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+67], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+68], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+69], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+70], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+71], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+72], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+73], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+74], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+75], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+12], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+13], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+14], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+15], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+16], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+17], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+18], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+19], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+20], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+21], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+22], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+23], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+24], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+25], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+26], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+27], acc143 // copy acc to vreg[143] +v_accvgpr_read_b32 v[vgprValuC+28], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+29], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+30], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+31], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+32], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+33], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+34], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+35], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+36], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+37], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+38], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+39], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+40], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+41], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+42], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+43], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+44], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+45], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+46], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+47], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+48], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+49], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+50], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+51], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+52], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+53], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+54], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+55], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+56], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+57], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+58], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+59], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+60], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+61], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+62], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+63], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+64], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+65], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+66], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+67], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+68], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+69], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+70], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+71], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+72], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+73], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+74], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+75], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+12], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+13], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+14], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+15], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+16], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+17], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+18], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+19], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+20], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+21], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+22], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+23], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+24], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+25], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+26], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+27], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+28], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+29], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+30], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+31], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+32], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+33], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+34], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+35], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+36], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+37], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+38], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+39], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+40], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+41], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+42], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+43], acc223 // copy acc to vreg[223] +v_accvgpr_read_b32 v[vgprValuC+44], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+45], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+46], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+47], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+48], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+49], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+50], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+51], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+52], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+53], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+54], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+55], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+56], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+57], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+58], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+59], acc239 // copy acc to vreg[239] +v_accvgpr_read_b32 v[vgprValuC+60], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+61], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+62], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+63], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+64], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+65], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+66], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+67], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+68], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+69], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+70], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+71], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+72], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+73], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+74], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+75], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[16:19], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[20:23], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[24:27], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[28:31], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[32:35], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[36:39], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[40:43], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[44:47], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[48:51], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[52:55], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[56:59], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_mul_i32 s12, s[sgprStrideD1J], 52 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[60:63], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[64:67], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[68:71], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_lshl_b32 s12, s[sgprStrideD1J], 2 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s12 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx4 v[72:75], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_1 // jump to end +label_GW_B0_E1_N: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v82, BufferOOB +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+12], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+13], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+14], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+15], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+16], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+17], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+18], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+19], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+24], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+25], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+26], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+27], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+28], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+29], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+30], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+31], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+32], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+33], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+34], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+35], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+36], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+37], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+38], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+39], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+44], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+45], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+46], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+47], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+48], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+49], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+50], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+51], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+52], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+53], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+54], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+55], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+56], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+57], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+58], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+59], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+64], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+65], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+66], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+67], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+68], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+69], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+70], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+71], acc47 // copy acc to vreg[47] +v_accvgpr_read_b32 v[vgprValuC+72], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+73], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+74], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+75], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+76], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+77], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+78], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+79], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+84], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+85], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+86], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+87], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+88], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+89], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+90], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+91], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v82, BufferOOB +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+12], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+13], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+14], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+15], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+16], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+17], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+18], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+19], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+24], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+25], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+26], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+27], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+28], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+29], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+30], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+31], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+32], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+33], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+34], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+35], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+36], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+37], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+38], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+39], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+44], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+45], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+46], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+47], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+48], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+49], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+50], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+51], acc95 // copy acc to vreg[95] +v_accvgpr_read_b32 v[vgprValuC+52], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+53], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+54], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+55], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+56], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+57], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+58], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+59], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+64], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+65], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+66], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+67], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+68], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+69], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+70], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+71], acc111 // copy acc to vreg[111] +v_accvgpr_read_b32 v[vgprValuC+72], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+73], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+74], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+75], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+76], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+77], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+78], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+79], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+84], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+85], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+86], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+87], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+88], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+89], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+90], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+91], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v82, BufferOOB +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+12], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+13], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+14], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+15], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+16], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+17], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+18], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+19], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+24], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+25], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+26], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+27], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+28], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+29], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+30], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+31], acc143 // copy acc to vreg[143] +v_accvgpr_read_b32 v[vgprValuC+32], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+33], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+34], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+35], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+36], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+37], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+38], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+39], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+44], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+45], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+46], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+47], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+48], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+49], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+50], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+51], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+52], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+53], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+54], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+55], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+56], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+57], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+58], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+59], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+64], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+65], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+66], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+67], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+68], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+69], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+70], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+71], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+72], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+73], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+74], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+75], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+76], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+77], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+78], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+79], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+84], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+85], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+86], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+87], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+88], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+89], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+90], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+91], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v82, BufferOOB +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v82, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v11, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v11, v82, v11, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v82, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v21, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v21, v82, v21, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v82, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v23, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v23, v82, v23, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v82, v40, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v41, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v82, v41, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v42, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v42, v82, v42, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v43, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v43, v82, v43, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v60, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v60, v82, v60, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v61, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v82, v61, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v62, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v82, v62, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v63, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v63, v82, v63, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v80, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v80, v82, v80, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v81, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v82, v81, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+12], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+13], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+14], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+15], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+16], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+17], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+18], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+19], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+24], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+25], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+26], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+27], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+28], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+29], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+30], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+31], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+32], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+33], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+34], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+35], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+36], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+37], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+38], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+39], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+44], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+45], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+46], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+47], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+48], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+49], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+50], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+51], acc223 // copy acc to vreg[223] +v_accvgpr_read_b32 v[vgprValuC+52], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+53], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+54], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+55], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+56], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+57], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+58], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+59], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+64], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+65], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+66], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+67], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+68], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+69], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+70], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+71], acc239 // copy acc to vreg[239] +v_accvgpr_read_b32 v[vgprValuC+72], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+73], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+74], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+75], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+76], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+77], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+78], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+79], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+84], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+85], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+86], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+87], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+88], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+89], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+90], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+91], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dwordx4 v[12:15], v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[16:19], v11, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[24:27], v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[28:31], v21, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[32:35], v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[36:39], v23, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[44:47], v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[48:51], v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[52:55], v42, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[56:59], v43, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[64:67], v60, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[68:71], v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[72:75], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[76:79], v63, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[84:87], v80, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dwordx4 v[88:91], v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_1 // jump to end +label_GW_B0_E1_M: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw1); (0,0,0,1:vw1); (0,0,0,2:vw1); (0,0,0,3:vw1); (0,0,1,0:vw1); (0,0,1,1:vw1); (0,0,1,2:vw1); (0,0,1,3:vw1); (0,0,2,0:vw1); (0,0,2,1:vw1); (0,0,2,2:vw1); (0,0,2,3:vw1); (0,0,3,0:vw1); (0,0,3,1:vw1); (0,0,3,2:vw1); (0,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+13], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+15], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+17], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+19], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+21], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+23], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+25], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+27], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+29], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+31], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+33], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+35], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+37], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+39], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+41], acc15 // copy acc to vreg[15] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (1,0,0,0:vw1); (1,0,0,1:vw1); (1,0,0,2:vw1); (1,0,0,3:vw1); (1,0,1,0:vw1); (1,0,1,1:vw1); (1,0,1,2:vw1); (1,0,1,3:vw1); (1,0,2,0:vw1); (1,0,2,1:vw1); (1,0,2,2:vw1); (1,0,2,3:vw1); (1,0,3,0:vw1); (1,0,3,1:vw1); (1,0,3,2:vw1); (1,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+13], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+15], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+17], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+19], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+21], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+23], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+25], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+27], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+29], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+31], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+33], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+35], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+37], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+39], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+41], acc31 // copy acc to vreg[31] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (2,0,0,0:vw1); (2,0,0,1:vw1); (2,0,0,2:vw1); (2,0,0,3:vw1); (2,0,1,0:vw1); (2,0,1,1:vw1); (2,0,1,2:vw1); (2,0,1,3:vw1); (2,0,2,0:vw1); (2,0,2,1:vw1); (2,0,2,2:vw1); (2,0,2,3:vw1); (2,0,3,0:vw1); (2,0,3,1:vw1); (2,0,3,2:vw1); (2,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+13], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+15], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+17], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+19], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+21], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+23], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+25], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+27], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+29], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+31], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+33], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+35], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+37], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+39], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+41], acc47 // copy acc to vreg[47] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw1); (3,0,0,1:vw1); (3,0,0,2:vw1); (3,0,0,3:vw1); (3,0,1,0:vw1); (3,0,1,1:vw1); (3,0,1,2:vw1); (3,0,1,3:vw1); (3,0,2,0:vw1); (3,0,2,1:vw1); (3,0,2,2:vw1); (3,0,2,3:vw1); (3,0,3,0:vw1); (3,0,3,1:vw1); (3,0,3,2:vw1); (3,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+13], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+15], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+17], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+19], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+21], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+23], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+25], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+27], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+29], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+31], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+33], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+35], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+37], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+39], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+41], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw1); (4,0,0,1:vw1); (4,0,0,2:vw1); (4,0,0,3:vw1); (4,0,1,0:vw1); (4,0,1,1:vw1); (4,0,1,2:vw1); (4,0,1,3:vw1); (4,0,2,0:vw1); (4,0,2,1:vw1); (4,0,2,2:vw1); (4,0,2,3:vw1); (4,0,3,0:vw1); (4,0,3,1:vw1); (4,0,3,2:vw1); (4,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+13], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+15], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+17], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+19], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+21], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+23], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+25], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+27], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+29], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+31], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+33], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+35], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+37], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+39], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+41], acc79 // copy acc to vreg[79] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 0, 1), (4, 0, 0, 2), (4, 0, 0, 3), (4, 0, 1, 0), (4, 0, 1, 1), (4, 0, 1, 2), (4, 0, 1, 3), (4, 0, 2, 0), (4, 0, 2, 1), (4, 0, 2, 2), (4, 0, 2, 3), (4, 0, 3, 0), (4, 0, 3, 1), (4, 0, 3, 2), (4, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #5 (d1,d0,vc1,vc0) = */ +/* (5,0,0,0:vw1); (5,0,0,1:vw1); (5,0,0,2:vw1); (5,0,0,3:vw1); (5,0,1,0:vw1); (5,0,1,1:vw1); (5,0,1,2:vw1); (5,0,1,3:vw1); (5,0,2,0:vw1); (5,0,2,1:vw1); (5,0,2,2:vw1); (5,0,2,3:vw1); (5,0,3,0:vw1); (5,0,3,1:vw1); (5,0,3,2:vw1); (5,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+13], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+15], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+17], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+19], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+21], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+23], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+25], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+27], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+29], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+31], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+33], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+35], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+37], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+39], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+41], acc95 // copy acc to vreg[95] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(5, 0, 0, 0), (5, 0, 0, 1), (5, 0, 0, 2), (5, 0, 0, 3), (5, 0, 1, 0), (5, 0, 1, 1), (5, 0, 1, 2), (5, 0, 1, 3), (5, 0, 2, 0), (5, 0, 2, 1), (5, 0, 2, 2), (5, 0, 2, 3), (5, 0, 3, 0), (5, 0, 3, 1), (5, 0, 3, 2), (5, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #6 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw1); (6,0,0,1:vw1); (6,0,0,2:vw1); (6,0,0,3:vw1); (6,0,1,0:vw1); (6,0,1,1:vw1); (6,0,1,2:vw1); (6,0,1,3:vw1); (6,0,2,0:vw1); (6,0,2,1:vw1); (6,0,2,2:vw1); (6,0,2,3:vw1); (6,0,3,0:vw1); (6,0,3,1:vw1); (6,0,3,2:vw1); (6,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+13], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+15], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+17], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+19], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+21], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+23], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+25], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+27], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+29], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+31], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+33], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+35], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+37], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+39], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+41], acc111 // copy acc to vreg[111] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 0, 1), (6, 0, 0, 2), (6, 0, 0, 3), (6, 0, 1, 0), (6, 0, 1, 1), (6, 0, 1, 2), (6, 0, 1, 3), (6, 0, 2, 0), (6, 0, 2, 1), (6, 0, 2, 2), (6, 0, 2, 3), (6, 0, 3, 0), (6, 0, 3, 1), (6, 0, 3, 2), (6, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #7 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw1); (7,0,0,1:vw1); (7,0,0,2:vw1); (7,0,0,3:vw1); (7,0,1,0:vw1); (7,0,1,1:vw1); (7,0,1,2:vw1); (7,0,1,3:vw1); (7,0,2,0:vw1); (7,0,2,1:vw1); (7,0,2,2:vw1); (7,0,2,3:vw1); (7,0,3,0:vw1); (7,0,3,1:vw1); (7,0,3,2:vw1); (7,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+13], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+15], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+17], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+19], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+21], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+23], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+25], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+27], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+29], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+31], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+33], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+35], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+37], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+39], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+41], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 0, 1), (7, 0, 0, 2), (7, 0, 0, 3), (7, 0, 1, 0), (7, 0, 1, 1), (7, 0, 1, 2), (7, 0, 1, 3), (7, 0, 2, 0), (7, 0, 2, 1), (7, 0, 2, 2), (7, 0, 2, 3), (7, 0, 3, 0), (7, 0, 3, 1), (7, 0, 3, 2), (7, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #8 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw1); (8,0,0,1:vw1); (8,0,0,2:vw1); (8,0,0,3:vw1); (8,0,1,0:vw1); (8,0,1,1:vw1); (8,0,1,2:vw1); (8,0,1,3:vw1); (8,0,2,0:vw1); (8,0,2,1:vw1); (8,0,2,2:vw1); (8,0,2,3:vw1); (8,0,3,0:vw1); (8,0,3,1:vw1); (8,0,3,2:vw1); (8,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+13], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+15], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+17], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+19], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+21], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+23], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+25], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+27], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+29], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+31], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+33], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+35], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+37], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+39], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+41], acc143 // copy acc to vreg[143] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 0, 1), (8, 0, 0, 2), (8, 0, 0, 3), (8, 0, 1, 0), (8, 0, 1, 1), (8, 0, 1, 2), (8, 0, 1, 3), (8, 0, 2, 0), (8, 0, 2, 1), (8, 0, 2, 2), (8, 0, 2, 3), (8, 0, 3, 0), (8, 0, 3, 1), (8, 0, 3, 2), (8, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #9 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw1); (9,0,0,1:vw1); (9,0,0,2:vw1); (9,0,0,3:vw1); (9,0,1,0:vw1); (9,0,1,1:vw1); (9,0,1,2:vw1); (9,0,1,3:vw1); (9,0,2,0:vw1); (9,0,2,1:vw1); (9,0,2,2:vw1); (9,0,2,3:vw1); (9,0,3,0:vw1); (9,0,3,1:vw1); (9,0,3,2:vw1); (9,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+13], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+15], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+17], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+19], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+21], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+23], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+25], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+27], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+29], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+31], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+33], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+35], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+37], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+39], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+41], acc159 // copy acc to vreg[159] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 0, 1), (9, 0, 0, 2), (9, 0, 0, 3), (9, 0, 1, 0), (9, 0, 1, 1), (9, 0, 1, 2), (9, 0, 1, 3), (9, 0, 2, 0), (9, 0, 2, 1), (9, 0, 2, 2), (9, 0, 2, 3), (9, 0, 3, 0), (9, 0, 3, 1), (9, 0, 3, 2), (9, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #10 (d1,d0,vc1,vc0) = */ +/* (10,0,0,0:vw1); (10,0,0,1:vw1); (10,0,0,2:vw1); (10,0,0,3:vw1); (10,0,1,0:vw1); (10,0,1,1:vw1); (10,0,1,2:vw1); (10,0,1,3:vw1); (10,0,2,0:vw1); (10,0,2,1:vw1); (10,0,2,2:vw1); (10,0,2,3:vw1); (10,0,3,0:vw1); (10,0,3,1:vw1); (10,0,3,2:vw1); (10,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+13], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+15], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+17], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+19], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+21], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+23], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+25], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+27], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+29], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+31], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+33], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+35], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+37], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+39], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+41], acc175 // copy acc to vreg[175] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(10, 0, 0, 0), (10, 0, 0, 1), (10, 0, 0, 2), (10, 0, 0, 3), (10, 0, 1, 0), (10, 0, 1, 1), (10, 0, 1, 2), (10, 0, 1, 3), (10, 0, 2, 0), (10, 0, 2, 1), (10, 0, 2, 2), (10, 0, 2, 3), (10, 0, 3, 0), (10, 0, 3, 1), (10, 0, 3, 2), (10, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #11 (d1,d0,vc1,vc0) = */ +/* (11,0,0,0:vw1); (11,0,0,1:vw1); (11,0,0,2:vw1); (11,0,0,3:vw1); (11,0,1,0:vw1); (11,0,1,1:vw1); (11,0,1,2:vw1); (11,0,1,3:vw1); (11,0,2,0:vw1); (11,0,2,1:vw1); (11,0,2,2:vw1); (11,0,2,3:vw1); (11,0,3,0:vw1); (11,0,3,1:vw1); (11,0,3,2:vw1); (11,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+13], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+15], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+17], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+19], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+21], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+23], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+25], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+27], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+29], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+31], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+33], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+35], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+37], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+39], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+41], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(11, 0, 0, 0), (11, 0, 0, 1), (11, 0, 0, 2), (11, 0, 0, 3), (11, 0, 1, 0), (11, 0, 1, 1), (11, 0, 1, 2), (11, 0, 1, 3), (11, 0, 2, 0), (11, 0, 2, 1), (11, 0, 2, 2), (11, 0, 2, 3), (11, 0, 3, 0), (11, 0, 3, 1), (11, 0, 3, 2), (11, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #12 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw1); (12,0,0,1:vw1); (12,0,0,2:vw1); (12,0,0,3:vw1); (12,0,1,0:vw1); (12,0,1,1:vw1); (12,0,1,2:vw1); (12,0,1,3:vw1); (12,0,2,0:vw1); (12,0,2,1:vw1); (12,0,2,2:vw1); (12,0,2,3:vw1); (12,0,3,0:vw1); (12,0,3,1:vw1); (12,0,3,2:vw1); (12,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+13], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+15], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+17], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+19], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+21], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+23], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+25], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+27], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+29], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+31], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+33], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+35], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+37], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+39], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+41], acc207 // copy acc to vreg[207] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 0, 1), (12, 0, 0, 2), (12, 0, 0, 3), (12, 0, 1, 0), (12, 0, 1, 1), (12, 0, 1, 2), (12, 0, 1, 3), (12, 0, 2, 0), (12, 0, 2, 1), (12, 0, 2, 2), (12, 0, 2, 3), (12, 0, 3, 0), (12, 0, 3, 1), (12, 0, 3, 2), (12, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #13 (d1,d0,vc1,vc0) = */ +/* (13,0,0,0:vw1); (13,0,0,1:vw1); (13,0,0,2:vw1); (13,0,0,3:vw1); (13,0,1,0:vw1); (13,0,1,1:vw1); (13,0,1,2:vw1); (13,0,1,3:vw1); (13,0,2,0:vw1); (13,0,2,1:vw1); (13,0,2,2:vw1); (13,0,2,3:vw1); (13,0,3,0:vw1); (13,0,3,1:vw1); (13,0,3,2:vw1); (13,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+13], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+15], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+17], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+19], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+21], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+23], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+25], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+27], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+29], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+31], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+33], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+35], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+37], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+39], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+41], acc223 // copy acc to vreg[223] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(13, 0, 0, 0), (13, 0, 0, 1), (13, 0, 0, 2), (13, 0, 0, 3), (13, 0, 1, 0), (13, 0, 1, 1), (13, 0, 1, 2), (13, 0, 1, 3), (13, 0, 2, 0), (13, 0, 2, 1), (13, 0, 2, 2), (13, 0, 2, 3), (13, 0, 3, 0), (13, 0, 3, 1), (13, 0, 3, 2), (13, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #14 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw1); (14,0,0,1:vw1); (14,0,0,2:vw1); (14,0,0,3:vw1); (14,0,1,0:vw1); (14,0,1,1:vw1); (14,0,1,2:vw1); (14,0,1,3:vw1); (14,0,2,0:vw1); (14,0,2,1:vw1); (14,0,2,2:vw1); (14,0,2,3:vw1); (14,0,3,0:vw1); (14,0,3,1:vw1); (14,0,3,2:vw1); (14,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+13], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+15], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+17], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+19], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+21], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+23], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+25], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+27], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+29], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+31], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+33], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+35], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+37], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+39], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+41], acc239 // copy acc to vreg[239] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 0, 1), (14, 0, 0, 2), (14, 0, 0, 3), (14, 0, 1, 0), (14, 0, 1, 1), (14, 0, 1, 2), (14, 0, 1, 3), (14, 0, 2, 0), (14, 0, 2, 1), (14, 0, 2, 2), (14, 0, 2, 3), (14, 0, 3, 0), (14, 0, 3, 1), (14, 0, 3, 2), (14, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #15 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw1); (15,0,0,1:vw1); (15,0,0,2:vw1); (15,0,0,3:vw1); (15,0,1,0:vw1); (15,0,1,1:vw1); (15,0,1,2:vw1); (15,0,1,3:vw1); (15,0,2,0:vw1); (15,0,2,1:vw1); (15,0,2,2:vw1); (15,0,2,3:vw1); (15,0,3,0:vw1); (15,0,3,1:vw1); (15,0,3,2:vw1); (15,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v42, BufferOOB +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s58, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s58 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s58, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s58 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v10, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v10, v42, v10, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v12, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v12, v42, v12, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v14, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v14, v42, v14, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v16, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v16, v42, v16, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v18, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v42, v18, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v20, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v42, v20, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v22, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v22, v42, v22, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v24, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v24, v42, v24, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v26, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v26, v42, v26, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v28, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v42, v28, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v30, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v30, v42, v30, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v32, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v32, v42, v32, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[58:59], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v34, v3, v0, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v42, v34, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v36, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v36, v42, v36, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v38, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v42, v38, s[62:63] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[58:59], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[62:63], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[62:63], s[58:59], s[62:63] // in0 && in1 +v_add_lshl_u32 v40, v3, v4, 0x2 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v40, v42, v40, s[62:63] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+11], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+13], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+15], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+17], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+19], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+21], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+23], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+25], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+27], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+29], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+31], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+33], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+35], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+37], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+39], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+41], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 0, 1), (15, 0, 0, 2), (15, 0, 0, 3), (15, 0, 1, 0), (15, 0, 1, 1), (15, 0, 1, 2), (15, 0, 1, 3), (15, 0, 2, 0), (15, 0, 2, 1), (15, 0, 2, 2), (15, 0, 2, 3), (15, 0, 3, 0), (15, 0, 3, 1), (15, 0, 3, 2), (15, 0, 3, 3)] */ + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v7, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v8, 0x7fff0000 // fp32 Nan +v_mov_b32 v9, 0x7fff // rounding bias for bfloat16 +buffer_store_dword v11, v10, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v13, v12, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v15, v14, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v17, v16, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v19, v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v21, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v23, v22, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v25, v24, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v27, v26, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v29, v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v31, v30, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v33, v32, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v35, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v37, v36, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v39, v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +buffer_store_dword v41, v40, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_1 // jump to end +label_GW_End_1: +s_getpc_b64 s[58:59] // addr of next instr +s_add_i32 s60, label_KernelEnd, 0x4 // target branch offset +s_add_u32 s58, s58, s60 // add target branch offset +s_addc_u32 s59, s59, 0 // add high and carry +s_setpc_b64 s[58:59] // branch to label_KernelEnd +label_GSU_5: +s_mov_b32 s[sgprSrdScaleAlphaVec+0], s[sgprAddressScaleAlphaVec+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdScaleAlphaVec+1], s[sgprAddressScaleAlphaVec+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdScaleAlphaVec+3], Srd127_96 // Set bits 127_96 in post-loop SRD +s_cmp_eq_u64 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], 0 // s[AddressScaleAlphaVec] == 0 ? +s_cbranch_scc0 label_ScaleAlphaVec_1AddrValid // branch if s[AddressScaleAlphaVec] != 0 +s_mov_b32 s[sgprSrdScaleAlphaVec+2], 0 +s_branch label_ScaleAlphaVec_1AddrValid_End +label_ScaleAlphaVec_1AddrValid: +s_mov_b32 s[sgprSrdScaleAlphaVec+2], s[sgprSizeI] +label_ScaleAlphaVec_1AddrValid_End: + +s_mul_i32 s[sgprSrdScaleAlphaVec+2], 0x4, s[sgprSrdScaleAlphaVec+2] // ScaleAlphaVec scaled by BPE +s_add_u32 s8, s[sgprWorkGroup2], 0x1 +s_mul_i32 s8, s[sgprBiasStride], s8 // stride * (wg+1) +s_cmp_eq_u32 s8, 0x0 // bias stride = 0? +s_cselect_b32 s8, s[sgprSizeI], s8 +s_mov_b32 s[sgprSrdBias+0], s[sgprAddressBias+0] // init SRD base address (lower) +s_mov_b32 s[sgprSrdBias+1], s[sgprAddressBias+1] // init SRD base address (upper) + other fields +s_mov_b32 s[sgprSrdBias+3], Srd127_96 // Set bits 127_96 in post-loop SRD +s_cmp_eq_u64 s[sgprAddressBias:sgprAddressBias+1], 0 // s[AddressBias] == 0 ? +s_cbranch_scc0 label_Bias_1AddrValid // branch if s[AddressBias] != 0 +s_mov_b32 s[sgprSrdBias+2], 0 +s_branch label_Bias_1AddrValid_End +label_Bias_1AddrValid: +s_mov_b32 s[sgprSrdBias+2], s8 +label_Bias_1AddrValid_End: + +label_Load_Biasf32_0_1: +s_cmpk_lg_u32 s[sgprBiasType], 0 // BiasType != 0 +s_cbranch_scc1 label_Load_Biasbf16_0_1 // Branch if true + +/******************************************/ +/* Read Bias to LDS */ +/******************************************/ +s_mul_i32 s[sgprSrdBias+2], 0x4, s[sgprSrdBias+2] // scaled by BPE +s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset +s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG +v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG +v_lshlrev_b32 v8, 0x2, v8 // Global bias address scaled by BPE +buffer_load_dword v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias +v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE +s_waitcnt vmcnt(0) // wait for bias load +s_barrier // Wait for all wavefronts +ds_write_b32 v8, v4 offset:0 // store bias +s_branch label_Load_Bias_End_1 // Branch to load bias end +label_Load_Biasbf16_0_1: +s_cmpk_lg_u32 s[sgprBiasType], 7 // BiasType != 7 +s_cbranch_scc1 label_Load_Bias_End_1 // Branch if true + +/******************************************/ +/* Read Bias to LDS */ +/******************************************/ +s_mul_i32 s[sgprSrdBias+2], 0x2, s[sgprSrdBias+2] // scaled by BPE +s_mul_i32 s8, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_add_u32 v8, s8, v[vgprSerial] // coord 0 = wgp0 * MT0 + thread offset +s_mul_i32 s8, s[sgprBiasStride], s[sgprWorkGroup2] // Stride * WG +v_add_u32 v8, s8, v8 // coord 0 = wgp0 * MT0 + thread offset + Stride * WG +v_lshlrev_b32 v8, 0x1, v8 // Global bias address scaled by BPE +buffer_load_short_d16 v4, v8, s[sgprSrdBias:sgprSrdBias+3], 0 offen offset:0 // load bias +v_lshlrev_b32 v8, 0x2, v[vgprSerial] // Local bias address scaled by BPE +s_waitcnt vmcnt(0) // wait for bias load +s_barrier // Wait for all wavefronts +v_lshlrev_b32 v4, 16, v4 // cvt bf16 to fp32. +ds_write_b32 v8, v4 offset:0 // store bias +s_branch label_Load_Bias_End_1 // Branch to load bias end +label_Load_Bias_End_1: +s_cmpk_eq_u32 s[sgprBeta], 0x0 // Beta == 0 +s_cbranch_scc0 label_GW_Beta_2 // Branch if Beta is not zero + +s_and_b32 s60, 255, s[sgprSizeI] // s60 = s[sgprSizeI] % 256 +s_add_u32 s61, -0x1, s[sgprNumWorkGroups0] +s_cmp_ge_u32 s[sgprWorkGroup0], s61 // wg0 >= nwg0-1 ? +s_cselect_b32 s60, s60, 0 // set rMT0 +s_cmpk_gt_u32 s60, 0x0 // rMT0 > 0 +s_cbranch_scc0 label_NoBranch_PFO42GJLMDBXSWVP_0 // Only branch on scc1 +// jump if edges required +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_B0_E1_M_1, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_B0_E1_M_1 +label_NoBranch_PFO42GJLMDBXSWVP_0: +s_and_b32 s60, 255, s[sgprSizeJ] // s60 = s[sgprSizeJ] % 256 +s_add_u32 s61, -0x1, s[sgprNumWorkGroups1] +s_cmp_ge_u32 s[sgprWorkGroup1], s61 // wg1 >= nwg1-1 +s_cselect_b32 s60, s60, 0 // set rMT1 +s_cmpk_gt_u32 s60, 0x0 // rMT1 > 0 +s_cbranch_scc0 label_NoBranch_XR5UBTMCR0HWLF5H_0 // Only branch on scc1 +// jump if edges required +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_B0_E1_N_1, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_B0_E1_N_1 +label_NoBranch_XR5UBTMCR0HWLF5H_0: +label_GW_B0_E0_2: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_0_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_0_edge_0 // Branch if true +label_To_Activation_None_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Abs_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Gelu_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Relu_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Sigmoid_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Tanh_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Geluscaling_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_To_Activation_Silu_VW4_1_beta_0_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_6 +label_ActivationSetPCAddrEnd_6: + +/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b128 v[20:23], v15 offset:0 // load bias +v_lshlrev_b32 v16, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_lshl_u32 v13, v3, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 +v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+32], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+33], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+34], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+35], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+36], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+37], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+38], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+39], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+40], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+41], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+42], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+43], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+44], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+45], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+46], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+47], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+48], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+49], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+50], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+51], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+52], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+53], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+54], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+55], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+56], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+57], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+58], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+59], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+60], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+61], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+62], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+63], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+64], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+65], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+66], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+67], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+68], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+69], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+70], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+71], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+72], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+73], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+74], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+75], acc47 // copy acc to vreg[47] +v_accvgpr_read_b32 v[vgprValuC+76], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+77], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+78], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+79], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+80], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+81], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+82], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+83], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+84], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+85], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+86], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+87], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+88], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+89], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+90], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+91], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #1 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+28], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+29], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+31], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+32], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+33], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+34], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+35], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+36], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+37], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+38], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+39], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+40], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+41], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+42], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+43], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+44], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+45], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+46], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+47], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+48], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+49], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+50], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+51], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+52], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+53], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+54], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+55], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+56], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+57], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+58], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+59], acc95 // copy acc to vreg[95] +v_accvgpr_read_b32 v[vgprValuC+60], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+61], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+62], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+63], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+64], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+65], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+66], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+67], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+68], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+69], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+70], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+71], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+72], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+73], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+74], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+75], acc111 // copy acc to vreg[111] +v_accvgpr_read_b32 v[vgprValuC+76], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+77], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+78], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+79], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+80], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+81], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+82], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+83], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+84], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+85], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+86], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+87], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+88], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+89], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+90], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+91], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #2 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+28], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+29], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+31], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+32], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+33], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+34], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+35], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+36], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+37], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+38], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+39], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+40], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+41], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+42], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+43], acc143 // copy acc to vreg[143] +v_accvgpr_read_b32 v[vgprValuC+44], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+45], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+46], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+47], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+48], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+49], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+50], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+51], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+52], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+53], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+54], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+55], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+56], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+57], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+58], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+59], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+60], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+61], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+62], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+63], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+64], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+65], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+66], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+67], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+68], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+69], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+70], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+71], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+72], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+73], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+74], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+75], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+76], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+77], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+78], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+79], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+80], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+81], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+82], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+83], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+84], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+85], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+86], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+87], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+88], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+89], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+90], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+91], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Batch #3 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+32], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+33], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+34], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+35], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+36], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+37], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+38], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+39], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+40], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+41], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+42], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+43], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+44], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+45], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+46], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+47], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+48], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+49], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+50], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+51], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+52], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+53], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+54], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+55], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+56], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+57], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+58], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+59], acc223 // copy acc to vreg[223] +v_accvgpr_read_b32 v[vgprValuC+60], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+61], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+62], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+63], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+64], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+65], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+66], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+67], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+68], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+69], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+70], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+71], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+72], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+73], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+74], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+75], acc239 // copy acc to vreg[239] +v_accvgpr_read_b32 v[vgprValuC+76], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+77], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+78], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+79], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+80], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+81], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+82], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+83], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+84], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+85], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+86], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+87], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+88], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+89], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+90], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+91], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt 0 // vmcnt(0) = 1 - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[24:25], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[26:27], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[32:33], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[44:45], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[56:57], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[24:25], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[26:27], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[68:69], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[80:81], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +// jump to end +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_End_2, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_End_2 +label_GW_B0_E1_N_1: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_0_edge_1 // Branch if true +label_To_Activation_None_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Abs_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Clippedrelu_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Gelu_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Leakyrelu_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Relu_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Sigmoid_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Tanh_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Geluscaling_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_To_Activation_Silu_VW4_1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_5 +label_ActivationSetPCAddrEnd_5: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=14 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v115, BufferOOB +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b128 v[16:19], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v29, v0, s60 +v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE +v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v36, v0, s60 +v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE +v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v39, v0, s60 +v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE +v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v0, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v0, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v64, v0, s60 +v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE +v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v0, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v0, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v0, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v95, v0, s60 +v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE +v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v102, v0, s60 +v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE +v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v113, v0, s60 +v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE +v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+24], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+25], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+26], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+27], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+32], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+33], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+34], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+35], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+40], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+41], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+42], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+43], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+48], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+49], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+50], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+51], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+52], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+53], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+54], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+55], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+60], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+61], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+62], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+63], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+68], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+69], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+70], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+71], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+76], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+77], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+78], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+79], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+80], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+81], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+82], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+83], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+88], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+89], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+90], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+91], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+96], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+97], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+98], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+99], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+104], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+105], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+106], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+107], acc47 // copy acc to vreg[47] +v_accvgpr_read_b32 v[vgprValuC+108], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+109], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+110], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+111], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+116], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+117], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+118], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+119], acc61 // copy acc to vreg[55] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0)] */ +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha +v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha +v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha +v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_mov_b32 v25, v5 +v_mov_b32 v26, v6 +v_mov_b32 v27, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan +v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] +v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan +v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] +v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword +buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v104, v4 +v_mov_b32 v105, v5 +v_mov_b32 v106, v6 +v_mov_b32 v107, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan +v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan +v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] +v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan +v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan +v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] +v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword +buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (3,0,2,0:vw4); (3,0,3,0:vw4); (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v115, BufferOOB +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[16:19], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v29, v0, s60 +v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE +v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v36, v0, s60 +v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE +v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v39, v0, s60 +v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE +v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v0, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v0, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v64, v0, s60 +v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE +v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v0, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v0, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v0, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v95, v0, s60 +v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE +v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v102, v0, s60 +v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE +v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v113, v0, s60 +v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE +v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+24], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+25], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+26], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+27], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+32], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+33], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+34], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+35], acc63 // copy acc to vreg[63] +v_accvgpr_read_b32 v[vgprValuC+40], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+41], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+42], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+43], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+48], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+49], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+50], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+51], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+52], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+53], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+54], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+55], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+60], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+61], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+62], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+63], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+68], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+69], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+70], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+71], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+76], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+77], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+78], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+79], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+80], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+81], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+82], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+83], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+88], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+89], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+90], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+91], acc95 // copy acc to vreg[95] +v_accvgpr_read_b32 v[vgprValuC+96], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+97], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+98], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+99], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+104], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+105], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+106], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+107], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+108], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+109], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+110], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+111], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+116], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+117], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+118], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+119], acc111 // copy acc to vreg[111] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(3, 0, 2, 0), (3, 0, 3, 0), (4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha +v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha +v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha +v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_mov_b32 v25, v5 +v_mov_b32 v26, v6 +v_mov_b32 v27, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan +v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] +v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan +v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] +v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword +buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v104, v4 +v_mov_b32 v105, v5 +v_mov_b32 v106, v6 +v_mov_b32 v107, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan +v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan +v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] +v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan +v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan +v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] +v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword +buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4); (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v115, BufferOOB +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[16:19], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v29, v0, s60 +v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE +v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v36, v0, s60 +v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE +v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v39, v0, s60 +v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE +v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v0, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v0, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v64, v0, s60 +v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE +v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v0, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v0, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v0, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v95, v0, s60 +v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE +v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v102, v0, s60 +v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE +v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v113, v0, s60 +v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE +v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+24], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+25], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+26], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+27], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+32], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+33], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+34], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+35], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+40], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+41], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+42], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+43], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+48], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+49], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+50], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+51], acc127 // copy acc to vreg[127] +v_accvgpr_read_b32 v[vgprValuC+52], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+53], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+54], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+55], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+60], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+61], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+62], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+63], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+68], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+69], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+70], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+71], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+76], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+77], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+78], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+79], acc143 // copy acc to vreg[143] +v_accvgpr_read_b32 v[vgprValuC+80], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+81], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+82], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+83], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+88], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+89], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+90], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+91], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+96], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+97], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+98], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+99], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+104], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+105], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+106], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+107], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+108], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+109], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+110], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+111], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+116], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+117], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+118], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+119], acc173 // copy acc to vreg[167] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0), (8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0)] */ +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha +v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha +v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha +v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_mov_b32 v25, v5 +v_mov_b32 v26, v6 +v_mov_b32 v27, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan +v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] +v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan +v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] +v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword +buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v104, v4 +v_mov_b32 v105, v5 +v_mov_b32 v106, v6 +v_mov_b32 v107, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan +v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan +v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] +v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan +v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan +v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] +v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword +buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4); (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v115, BufferOOB +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v115, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[16:19], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v115, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v29, v0, s60 +v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE +v_cndmask_b32 v29, v115, v29, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v115, v28, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v36, v0, s60 +v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE +v_cndmask_b32 v36, v115, v36, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v115, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v39, v0, s60 +v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE +v_cndmask_b32 v39, v115, v39, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v115, v38, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v0, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v115, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v115, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v0, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v115, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v115, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v64, v0, s60 +v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE +v_cndmask_b32 v64, v115, v64, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v59, v115, v59, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v0, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v115, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v115, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v0, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v115, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v115, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v115, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v115, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v0, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v115, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v115, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v95, v0, s60 +v_lshlrev_b32 v95, 0x2, v95 // Bias address scaled by BPE +v_cndmask_b32 v95, v115, v95, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v100, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v94, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v94, v115, v94, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v102, v0, s60 +v_lshlrev_b32 v102, 0x2, v102 // Bias address scaled by BPE +v_cndmask_b32 v102, v115, v102, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v103, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v101, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v101, v115, v101, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v113, v0, s60 +v_lshlrev_b32 v113, 0x2, v113 // Bias address scaled by BPE +v_cndmask_b32 v113, v115, v113, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v114, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v112, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v112, v115, v112, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+24], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+25], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+26], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+27], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+32], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+33], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+34], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+35], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+40], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+41], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+42], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+43], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+48], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+49], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+50], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+51], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+52], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+53], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+54], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+55], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+60], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+61], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+62], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+63], acc191 // copy acc to vreg[191] +v_accvgpr_read_b32 v[vgprValuC+68], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+69], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+70], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+71], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+76], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+77], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+78], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+79], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+80], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+81], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+82], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+83], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+88], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+89], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+90], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+91], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+96], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+97], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+98], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+99], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+104], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+105], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+106], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+107], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+108], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+109], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+110], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+111], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+116], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+117], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+118], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+119], acc223 // copy acc to vreg[223] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0), (12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+104], s[sgprAlpha], v[vgprValuC+104] // *= alpha +v_mul_f32 v[vgprValuC+105], s[sgprAlpha], v[vgprValuC+105] // *= alpha +v_mul_f32 v[vgprValuC+106], s[sgprAlpha], v[vgprValuC+106] // *= alpha +v_mul_f32 v[vgprValuC+107], s[sgprAlpha], v[vgprValuC+107] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_mov_b32 v25, v5 +v_mov_b32 v26, v6 +v_mov_b32 v27, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan +v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] +v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan +v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] +v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword +buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +buffer_store_dwordx2 v[88:89], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +buffer_store_dwordx2 v[96:97], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+104:vgprValuC+104+1], v[20:21], v[vgprValuC+104:vgprValuC+104+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+106:vgprValuC+106+1], v[22:23], v[vgprValuC+106:vgprValuC+106+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+104:vgprValuC+104+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+106:vgprValuC+106+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v104, v4 +v_mov_b32 v105, v5 +v_mov_b32 v106, v6 +v_mov_b32 v107, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+104], v[vgprValuC+104] // check Nan +v_bfe_u32 v9, v[vgprValuC+104], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+104], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+104], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+104], 16, v[vgprValuC+104] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+105], v[vgprValuC+105] // check Nan +v_bfe_u32 v9, v[vgprValuC+105], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+105], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+105], v9, v11, s[60:61] +v_and_or_b32 v104, v[vgprValuC+105], v10, v[vgprValuC+104] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+106], v[vgprValuC+106] // check Nan +v_bfe_u32 v9, v[vgprValuC+106], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+106], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+106], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+106], 16, v[vgprValuC+106] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+107], v[vgprValuC+107] // check Nan +v_bfe_u32 v9, v[vgprValuC+107], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+107], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+107], v9, v11, s[60:61] +v_and_or_b32 v105, v[vgprValuC+107], v10, v[vgprValuC+106] // pack two bf16 to dword +buffer_store_dwordx2 v[104:105], v94, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v101, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v112, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v73, BufferOOB +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v73, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[16:19], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[20:23], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v73, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v29, v0, s60 +v_lshlrev_b32 v29, 0x2, v29 // Bias address scaled by BPE +v_cndmask_b32 v29, v73, v29, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v30, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v28, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v28, v73, v28, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v36, v0, s60 +v_lshlrev_b32 v36, 0x2, v36 // Bias address scaled by BPE +v_cndmask_b32 v36, v73, v36, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v37, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v31, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v73, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v39, v0, s60 +v_lshlrev_b32 v39, 0x2, v39 // Bias address scaled by BPE +v_cndmask_b32 v39, v73, v39, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v44, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v38, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v38, v73, v38, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v0, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v73, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v73, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v0, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v73, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v73, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v64, v0, s60 +v_lshlrev_b32 v64, 0x2, v64 // Bias address scaled by BPE +v_cndmask_b32 v64, v73, v64, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v65, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v59, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v59, v73, v59, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v0, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v73, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v72, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v73, v66, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+24], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+25], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+26], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+27], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+32], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+33], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+34], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+35], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+40], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+41], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+42], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+43], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+48], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+49], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+50], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+51], acc239 // copy acc to vreg[239] +v_accvgpr_read_b32 v[vgprValuC+52], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+53], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+54], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+55], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+60], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+61], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+62], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+63], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+68], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+69], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+70], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+71], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+76], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+77], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+78], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+79], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+25], s[sgprAlpha], v[vgprValuC+25] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+27], s[sgprAlpha], v[vgprValuC+27] // *= alpha +v_mul_f32 v[vgprValuC+32], s[sgprAlpha], v[vgprValuC+32] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+34], s[sgprAlpha], v[vgprValuC+34] // *= alpha +v_mul_f32 v[vgprValuC+35], s[sgprAlpha], v[vgprValuC+35] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+69], s[sgprAlpha], v[vgprValuC+69] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+71], s[sgprAlpha], v[vgprValuC+71] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+24:vgprValuC+24+1], v[20:21], v[vgprValuC+24:vgprValuC+24+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+26:vgprValuC+26+1], v[22:23], v[vgprValuC+26:vgprValuC+26+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+24:vgprValuC+24+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+26:vgprValuC+26+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_mov_b32 v25, v5 +v_mov_b32 v26, v6 +v_mov_b32 v27, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+24], 16, v[vgprValuC+24] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+25], v[vgprValuC+25] // check Nan +v_bfe_u32 v9, v[vgprValuC+25], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+25], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+25], v9, v11, s[60:61] +v_and_or_b32 v24, v[vgprValuC+25], v10, v[vgprValuC+24] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+26], 16, v[vgprValuC+26] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+27], v[vgprValuC+27] // check Nan +v_bfe_u32 v9, v[vgprValuC+27], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+27], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+27], v9, v11, s[60:61] +v_and_or_b32 v25, v[vgprValuC+27], v10, v[vgprValuC+26] // pack two bf16 to dword +buffer_store_dwordx2 v[24:25], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+32:vgprValuC+32+1], v[20:21], v[vgprValuC+32:vgprValuC+32+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+34:vgprValuC+34+1], v[22:23], v[vgprValuC+34:vgprValuC+34+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+32:vgprValuC+32+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+34:vgprValuC+34+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v32, v4 +v_mov_b32 v33, v5 +v_mov_b32 v34, v6 +v_mov_b32 v35, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+32], v[vgprValuC+32] // check Nan +v_bfe_u32 v9, v[vgprValuC+32], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+32], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+32], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+32], 16, v[vgprValuC+32] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_and_or_b32 v32, v[vgprValuC+33], v10, v[vgprValuC+32] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+34], v[vgprValuC+34] // check Nan +v_bfe_u32 v9, v[vgprValuC+34], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+34], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+34], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+34], 16, v[vgprValuC+34] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+35], v[vgprValuC+35] // check Nan +v_bfe_u32 v9, v[vgprValuC+35], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+35], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+35], v9, v11, s[60:61] +v_and_or_b32 v33, v[vgprValuC+35], v10, v[vgprValuC+34] // pack two bf16 to dword +buffer_store_dwordx2 v[32:33], v28, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +buffer_store_dwordx2 v[40:41], v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +buffer_store_dwordx2 v[48:49], v38, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +buffer_store_dwordx2 v[52:53], v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +buffer_store_dwordx2 v[60:61], v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+68:vgprValuC+68+1], v[20:21], v[vgprValuC+68:vgprValuC+68+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+70:vgprValuC+70+1], v[22:23], v[vgprValuC+70:vgprValuC+70+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+68:vgprValuC+68+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+70:vgprValuC+70+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_mov_b32 v69, v5 +v_mov_b32 v70, v6 +v_mov_b32 v71, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+68], 16, v[vgprValuC+68] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+69], v[vgprValuC+69] // check Nan +v_bfe_u32 v9, v[vgprValuC+69], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+69], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+69], v9, v11, s[60:61] +v_and_or_b32 v68, v[vgprValuC+69], v10, v[vgprValuC+68] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+70], 16, v[vgprValuC+70] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+71], v[vgprValuC+71] // check Nan +v_bfe_u32 v9, v[vgprValuC+71], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+71], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+71], v9, v11, s[60:61] +v_and_or_b32 v69, v[vgprValuC+71], v10, v[vgprValuC+70] // pack two bf16 to dword +buffer_store_dwordx2 v[68:69], v59, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v20, 1.0, v20, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v21, 1.0, v21, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(20)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v22, 1.0, v22, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(20)(2) +v_pk_add_f32 v[4:5], v[16:17], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[18:19], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +buffer_store_dwordx2 v[76:77], v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +// jump to end +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_End_2, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_End_2 +label_GW_B0_E1_M_1: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW1_beta_0_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW1_beta_0_edge_1 // Branch if true +label_To_Activation_None_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Abs_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Clippedrelu_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Gelu_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Leakyrelu_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Relu_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Sigmoid_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Tanh_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Geluscaling_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_To_Activation_Silu_VW1_beta_0_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_4 +label_ActivationSetPCAddrEnd_4: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw1); (0,0,0,1:vw1); (0,0,0,2:vw1); (0,0,0,3:vw1); (0,0,1,0:vw1); (0,0,1,1:vw1); (0,0,1,2:vw1); (0,0,1,3:vw1); (0,0,2,0:vw1); (0,0,2,1:vw1); (0,0,2,2:vw1); (0,0,2,3:vw1); (0,0,3,0:vw1); (0,0,3,1:vw1); (0,0,3,2:vw1); (0,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+24], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+36], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+40], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+44], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+48], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+52], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+56], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+60], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+64], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+68], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+72], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+76], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+80], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+84], acc15 // copy acc to vreg[15] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (1,0,0,0:vw1); (1,0,0,1:vw1); (1,0,0,2:vw1); (1,0,0,3:vw1); (1,0,1,0:vw1); (1,0,1,1:vw1); (1,0,1,2:vw1); (1,0,1,3:vw1); (1,0,2,0:vw1); (1,0,2,1:vw1); (1,0,2,2:vw1); (1,0,2,3:vw1); (1,0,3,0:vw1); (1,0,3,1:vw1); (1,0,3,2:vw1); (1,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+24], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+30], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+36], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+40], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+44], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+48], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+52], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+56], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+60], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+64], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+68], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+72], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+76], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+80], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+84], acc31 // copy acc to vreg[31] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (2,0,0,0:vw1); (2,0,0,1:vw1); (2,0,0,2:vw1); (2,0,0,3:vw1); (2,0,1,0:vw1); (2,0,1,1:vw1); (2,0,1,2:vw1); (2,0,1,3:vw1); (2,0,2,0:vw1); (2,0,2,1:vw1); (2,0,2,2:vw1); (2,0,2,3:vw1); (2,0,3,0:vw1); (2,0,3,1:vw1); (2,0,3,2:vw1); (2,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+24], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+30], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+36], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+40], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+44], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+48], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+52], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+56], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+60], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+64], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+68], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+72], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+76], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+80], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+84], acc47 // copy acc to vreg[47] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw1); (3,0,0,1:vw1); (3,0,0,2:vw1); (3,0,0,3:vw1); (3,0,1,0:vw1); (3,0,1,1:vw1); (3,0,1,2:vw1); (3,0,1,3:vw1); (3,0,2,0:vw1); (3,0,2,1:vw1); (3,0,2,2:vw1); (3,0,2,3:vw1); (3,0,3,0:vw1); (3,0,3,1:vw1); (3,0,3,2:vw1); (3,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+24], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+30], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+36], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+40], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+44], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+48], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+52], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+56], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+60], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+64], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+68], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+72], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+76], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+80], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+84], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw1); (4,0,0,1:vw1); (4,0,0,2:vw1); (4,0,0,3:vw1); (4,0,1,0:vw1); (4,0,1,1:vw1); (4,0,1,2:vw1); (4,0,1,3:vw1); (4,0,2,0:vw1); (4,0,2,1:vw1); (4,0,2,2:vw1); (4,0,2,3:vw1); (4,0,3,0:vw1); (4,0,3,1:vw1); (4,0,3,2:vw1); (4,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+24], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+36], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+40], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+44], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+48], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+52], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+56], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+60], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+64], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+68], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+72], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+76], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+80], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+84], acc79 // copy acc to vreg[79] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 0, 1), (4, 0, 0, 2), (4, 0, 0, 3), (4, 0, 1, 0), (4, 0, 1, 1), (4, 0, 1, 2), (4, 0, 1, 3), (4, 0, 2, 0), (4, 0, 2, 1), (4, 0, 2, 2), (4, 0, 2, 3), (4, 0, 3, 0), (4, 0, 3, 1), (4, 0, 3, 2), (4, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #5 (d1,d0,vc1,vc0) = */ +/* (5,0,0,0:vw1); (5,0,0,1:vw1); (5,0,0,2:vw1); (5,0,0,3:vw1); (5,0,1,0:vw1); (5,0,1,1:vw1); (5,0,1,2:vw1); (5,0,1,3:vw1); (5,0,2,0:vw1); (5,0,2,1:vw1); (5,0,2,2:vw1); (5,0,2,3:vw1); (5,0,3,0:vw1); (5,0,3,1:vw1); (5,0,3,2:vw1); (5,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+24], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+30], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+36], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+40], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+44], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+48], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+52], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+56], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+60], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+64], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+68], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+72], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+76], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+80], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+84], acc95 // copy acc to vreg[95] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(5, 0, 0, 0), (5, 0, 0, 1), (5, 0, 0, 2), (5, 0, 0, 3), (5, 0, 1, 0), (5, 0, 1, 1), (5, 0, 1, 2), (5, 0, 1, 3), (5, 0, 2, 0), (5, 0, 2, 1), (5, 0, 2, 2), (5, 0, 2, 3), (5, 0, 3, 0), (5, 0, 3, 1), (5, 0, 3, 2), (5, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #6 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw1); (6,0,0,1:vw1); (6,0,0,2:vw1); (6,0,0,3:vw1); (6,0,1,0:vw1); (6,0,1,1:vw1); (6,0,1,2:vw1); (6,0,1,3:vw1); (6,0,2,0:vw1); (6,0,2,1:vw1); (6,0,2,2:vw1); (6,0,2,3:vw1); (6,0,3,0:vw1); (6,0,3,1:vw1); (6,0,3,2:vw1); (6,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+24], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+30], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+36], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+40], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+44], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+48], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+52], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+56], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+60], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+64], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+68], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+72], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+76], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+80], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+84], acc111 // copy acc to vreg[111] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 0, 1), (6, 0, 0, 2), (6, 0, 0, 3), (6, 0, 1, 0), (6, 0, 1, 1), (6, 0, 1, 2), (6, 0, 1, 3), (6, 0, 2, 0), (6, 0, 2, 1), (6, 0, 2, 2), (6, 0, 2, 3), (6, 0, 3, 0), (6, 0, 3, 1), (6, 0, 3, 2), (6, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #7 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw1); (7,0,0,1:vw1); (7,0,0,2:vw1); (7,0,0,3:vw1); (7,0,1,0:vw1); (7,0,1,1:vw1); (7,0,1,2:vw1); (7,0,1,3:vw1); (7,0,2,0:vw1); (7,0,2,1:vw1); (7,0,2,2:vw1); (7,0,2,3:vw1); (7,0,3,0:vw1); (7,0,3,1:vw1); (7,0,3,2:vw1); (7,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+24], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+30], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+36], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+40], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+44], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+48], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+52], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+56], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+60], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+64], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+68], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+72], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+76], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+80], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+84], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 0, 1), (7, 0, 0, 2), (7, 0, 0, 3), (7, 0, 1, 0), (7, 0, 1, 1), (7, 0, 1, 2), (7, 0, 1, 3), (7, 0, 2, 0), (7, 0, 2, 1), (7, 0, 2, 2), (7, 0, 2, 3), (7, 0, 3, 0), (7, 0, 3, 1), (7, 0, 3, 2), (7, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #8 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw1); (8,0,0,1:vw1); (8,0,0,2:vw1); (8,0,0,3:vw1); (8,0,1,0:vw1); (8,0,1,1:vw1); (8,0,1,2:vw1); (8,0,1,3:vw1); (8,0,2,0:vw1); (8,0,2,1:vw1); (8,0,2,2:vw1); (8,0,2,3:vw1); (8,0,3,0:vw1); (8,0,3,1:vw1); (8,0,3,2:vw1); (8,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+24], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+36], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+40], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+44], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+48], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+52], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+56], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+60], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+64], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+68], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+72], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+76], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+80], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+84], acc143 // copy acc to vreg[143] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 0, 1), (8, 0, 0, 2), (8, 0, 0, 3), (8, 0, 1, 0), (8, 0, 1, 1), (8, 0, 1, 2), (8, 0, 1, 3), (8, 0, 2, 0), (8, 0, 2, 1), (8, 0, 2, 2), (8, 0, 2, 3), (8, 0, 3, 0), (8, 0, 3, 1), (8, 0, 3, 2), (8, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #9 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw1); (9,0,0,1:vw1); (9,0,0,2:vw1); (9,0,0,3:vw1); (9,0,1,0:vw1); (9,0,1,1:vw1); (9,0,1,2:vw1); (9,0,1,3:vw1); (9,0,2,0:vw1); (9,0,2,1:vw1); (9,0,2,2:vw1); (9,0,2,3:vw1); (9,0,3,0:vw1); (9,0,3,1:vw1); (9,0,3,2:vw1); (9,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+24], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+30], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+36], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+40], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+44], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+48], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+52], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+56], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+60], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+64], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+68], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+72], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+76], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+80], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+84], acc159 // copy acc to vreg[159] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 0, 1), (9, 0, 0, 2), (9, 0, 0, 3), (9, 0, 1, 0), (9, 0, 1, 1), (9, 0, 1, 2), (9, 0, 1, 3), (9, 0, 2, 0), (9, 0, 2, 1), (9, 0, 2, 2), (9, 0, 2, 3), (9, 0, 3, 0), (9, 0, 3, 1), (9, 0, 3, 2), (9, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #10 (d1,d0,vc1,vc0) = */ +/* (10,0,0,0:vw1); (10,0,0,1:vw1); (10,0,0,2:vw1); (10,0,0,3:vw1); (10,0,1,0:vw1); (10,0,1,1:vw1); (10,0,1,2:vw1); (10,0,1,3:vw1); (10,0,2,0:vw1); (10,0,2,1:vw1); (10,0,2,2:vw1); (10,0,2,3:vw1); (10,0,3,0:vw1); (10,0,3,1:vw1); (10,0,3,2:vw1); (10,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+24], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+30], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+36], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+40], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+44], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+48], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+52], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+56], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+60], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+64], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+68], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+72], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+76], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+80], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+84], acc175 // copy acc to vreg[175] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(10, 0, 0, 0), (10, 0, 0, 1), (10, 0, 0, 2), (10, 0, 0, 3), (10, 0, 1, 0), (10, 0, 1, 1), (10, 0, 1, 2), (10, 0, 1, 3), (10, 0, 2, 0), (10, 0, 2, 1), (10, 0, 2, 2), (10, 0, 2, 3), (10, 0, 3, 0), (10, 0, 3, 1), (10, 0, 3, 2), (10, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #11 (d1,d0,vc1,vc0) = */ +/* (11,0,0,0:vw1); (11,0,0,1:vw1); (11,0,0,2:vw1); (11,0,0,3:vw1); (11,0,1,0:vw1); (11,0,1,1:vw1); (11,0,1,2:vw1); (11,0,1,3:vw1); (11,0,2,0:vw1); (11,0,2,1:vw1); (11,0,2,2:vw1); (11,0,2,3:vw1); (11,0,3,0:vw1); (11,0,3,1:vw1); (11,0,3,2:vw1); (11,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+24], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+30], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+36], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+40], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+44], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+48], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+52], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+56], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+60], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+64], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+68], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+72], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+76], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+80], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+84], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(11, 0, 0, 0), (11, 0, 0, 1), (11, 0, 0, 2), (11, 0, 0, 3), (11, 0, 1, 0), (11, 0, 1, 1), (11, 0, 1, 2), (11, 0, 1, 3), (11, 0, 2, 0), (11, 0, 2, 1), (11, 0, 2, 2), (11, 0, 2, 3), (11, 0, 3, 0), (11, 0, 3, 1), (11, 0, 3, 2), (11, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #12 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw1); (12,0,0,1:vw1); (12,0,0,2:vw1); (12,0,0,3:vw1); (12,0,1,0:vw1); (12,0,1,1:vw1); (12,0,1,2:vw1); (12,0,1,3:vw1); (12,0,2,0:vw1); (12,0,2,1:vw1); (12,0,2,2:vw1); (12,0,2,3:vw1); (12,0,3,0:vw1); (12,0,3,1:vw1); (12,0,3,2:vw1); (12,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+24], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+36], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+40], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+44], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+48], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+52], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+56], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+60], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+64], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+68], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+72], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+76], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+80], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+84], acc207 // copy acc to vreg[207] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 0, 1), (12, 0, 0, 2), (12, 0, 0, 3), (12, 0, 1, 0), (12, 0, 1, 1), (12, 0, 1, 2), (12, 0, 1, 3), (12, 0, 2, 0), (12, 0, 2, 1), (12, 0, 2, 2), (12, 0, 2, 3), (12, 0, 3, 0), (12, 0, 3, 1), (12, 0, 3, 2), (12, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #13 (d1,d0,vc1,vc0) = */ +/* (13,0,0,0:vw1); (13,0,0,1:vw1); (13,0,0,2:vw1); (13,0,0,3:vw1); (13,0,1,0:vw1); (13,0,1,1:vw1); (13,0,1,2:vw1); (13,0,1,3:vw1); (13,0,2,0:vw1); (13,0,2,1:vw1); (13,0,2,2:vw1); (13,0,2,3:vw1); (13,0,3,0:vw1); (13,0,3,1:vw1); (13,0,3,2:vw1); (13,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+24], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+30], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+36], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+40], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+44], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+48], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+52], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+56], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+60], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+64], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+68], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+72], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+76], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+80], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+84], acc223 // copy acc to vreg[223] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(13, 0, 0, 0), (13, 0, 0, 1), (13, 0, 0, 2), (13, 0, 0, 3), (13, 0, 1, 0), (13, 0, 1, 1), (13, 0, 1, 2), (13, 0, 1, 3), (13, 0, 2, 0), (13, 0, 2, 1), (13, 0, 2, 2), (13, 0, 2, 3), (13, 0, 3, 0), (13, 0, 3, 1), (13, 0, 3, 2), (13, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #14 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw1); (14,0,0,1:vw1); (14,0,0,2:vw1); (14,0,0,3:vw1); (14,0,1,0:vw1); (14,0,1,1:vw1); (14,0,1,2:vw1); (14,0,1,3:vw1); (14,0,2,0:vw1); (14,0,2,1:vw1); (14,0,2,2:vw1); (14,0,2,3:vw1); (14,0,3,0:vw1); (14,0,3,1:vw1); (14,0,3,2:vw1); (14,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+24], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+30], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+36], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+40], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+44], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+48], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+52], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+56], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+60], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+64], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+68], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+72], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+76], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+80], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+84], acc239 // copy acc to vreg[239] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 0, 1), (14, 0, 0, 2), (14, 0, 0, 3), (14, 0, 1, 0), (14, 0, 1, 1), (14, 0, 1, 2), (14, 0, 1, 3), (14, 0, 2, 0), (14, 0, 2, 1), (14, 0, 2, 2), (14, 0, 2, 3), (14, 0, 3, 0), (14, 0, 3, 1), (14, 0, 3, 2), (14, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Edge Batch #15 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw1); (15,0,0,1:vw1); (15,0,0,2:vw1); (15,0,0,3:vw1); (15,0,1,0:vw1); (15,0,1,1:vw1); (15,0,1,2:vw1); (15,0,1,3:vw1); (15,0,2,0:vw1); (15,0,2,1:vw1); (15,0,2,2:vw1); (15,0,2,3:vw1); (15,0,3,0:vw1); (15,0,3,1:vw1); (15,0,3,2:vw1); (15,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v85, BufferOOB +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v85, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v16, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v17, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v85, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v20, v4, s60 +v_lshlrev_b32 v20, 0x2, v20 // Bias address scaled by BPE +v_cndmask_b32 v20, v85, v20, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v22, v20 offset:0 // load bias +v_lshlrev_b32 v21, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v23, v21, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v19, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v19, v85, v19, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v26, v4, s60 +v_lshlrev_b32 v26, 0x2, v26 // Bias address scaled by BPE +v_cndmask_b32 v26, v85, v26, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v28, v26 offset:0 // load bias +v_lshlrev_b32 v27, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v29, v27, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v25, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v25, v85, v25, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v32, v4, s60 +v_lshlrev_b32 v32, 0x2, v32 // Bias address scaled by BPE +v_cndmask_b32 v32, v85, v32, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v34, v32 offset:0 // load bias +v_lshlrev_b32 v33, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v35, v33, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v31, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v31, v85, v31, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v38, v0, s60 +v_lshlrev_b32 v38, 0x2, v38 // Bias address scaled by BPE +v_cndmask_b32 v38, v85, v38, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v39, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v37, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v37, v85, v37, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v4, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v85, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v85, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v46, v4, s60 +v_lshlrev_b32 v46, 0x2, v46 // Bias address scaled by BPE +v_cndmask_b32 v46, v85, v46, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v47, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v45, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v45, v85, v45, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v50, v4, s60 +v_lshlrev_b32 v50, 0x2, v50 // Bias address scaled by BPE +v_cndmask_b32 v50, v85, v50, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v51, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v49, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v49, v85, v49, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v85, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v53, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v53, v85, v53, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v58, v4, s60 +v_lshlrev_b32 v58, 0x2, v58 // Bias address scaled by BPE +v_cndmask_b32 v58, v85, v58, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v59, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v57, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v57, v85, v57, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v4, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v85, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v85, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v66, v4, s60 +v_lshlrev_b32 v66, 0x2, v66 // Bias address scaled by BPE +v_cndmask_b32 v66, v85, v66, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v67, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v65, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v65, v85, v65, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v70, v0, s60 +v_lshlrev_b32 v70, 0x2, v70 // Bias address scaled by BPE +v_cndmask_b32 v70, v85, v70, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v71, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v85, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v74, v4, s60 +v_lshlrev_b32 v74, 0x2, v74 // Bias address scaled by BPE +v_cndmask_b32 v74, v85, v74, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v75, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v73, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v73, v85, v73, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v78, v4, s60 +v_lshlrev_b32 v78, 0x2, v78 // Bias address scaled by BPE +v_cndmask_b32 v78, v85, v78, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v79, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v77, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v77, v85, v77, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v4, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v85, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v85, v81, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+18], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+24], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+30], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+36], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+40], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+44], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+48], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+52], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+56], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+60], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+64], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+68], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+72], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+76], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+80], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+84], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 0, 1), (15, 0, 0, 2), (15, 0, 0, 3), (15, 0, 1, 0), (15, 0, 1, 1), (15, 0, 1, 2), (15, 0, 1, 3), (15, 0, 2, 0), (15, 0, 2, 1), (15, 0, 2, 2), (15, 0, 2, 3), (15, 0, 3, 0), (15, 0, 3, 1), (15, 0, 3, 2), (15, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+18], s[sgprAlpha], v[vgprValuC+18] // *= alpha +v_mul_f32 v[vgprValuC+24], s[sgprAlpha], v[vgprValuC+24] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+68], s[sgprAlpha], v[vgprValuC+68] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +s_waitcnt 0 // wait for ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+18], v17, v[vgprValuC+18] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+18] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v18, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+18], v[vgprValuC+18] // check Nan +v_bfe_u32 v9, v[vgprValuC+18], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+18], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+18], v9, v11, s[60:61] +v_lshrrev_b32 v18, 16, v[vgprValuC+18] // convert C to bf16 +buffer_store_short v18, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+24], v23, v[vgprValuC+24] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+24] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v24, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+24], v[vgprValuC+24] // check Nan +v_bfe_u32 v9, v[vgprValuC+24], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+24], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+24], v9, v11, s[60:61] +v_lshrrev_b32 v24, 16, v[vgprValuC+24] // convert C to bf16 +buffer_store_short v24, v19, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+30], v29, v[vgprValuC+30] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+30] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v30, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v30, 16, v[vgprValuC+30] // convert C to bf16 +buffer_store_short v30, v25, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+36], v35, v[vgprValuC+36] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+36] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v36, 16, v[vgprValuC+36] // convert C to bf16 +buffer_store_short v36, v31, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v17, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v37, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+44], v23, v[vgprValuC+44] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+44] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v44, 16, v[vgprValuC+44] // convert C to bf16 +buffer_store_short v44, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+48], v29, v[vgprValuC+48] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+48] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v48, 16, v[vgprValuC+48] // convert C to bf16 +buffer_store_short v48, v45, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+52], v35, v[vgprValuC+52] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+52] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v52, 16, v[vgprValuC+52] // convert C to bf16 +buffer_store_short v52, v49, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+56], v17, v[vgprValuC+56] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+56] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v56, 16, v[vgprValuC+56] // convert C to bf16 +buffer_store_short v56, v53, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v23, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v57, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+64], v29, v[vgprValuC+64] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+64] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v64, 16, v[vgprValuC+64] // convert C to bf16 +buffer_store_short v64, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+68], v35, v[vgprValuC+68] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+68] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v68, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+68], v[vgprValuC+68] // check Nan +v_bfe_u32 v9, v[vgprValuC+68], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+68], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+68], v9, v11, s[60:61] +v_lshrrev_b32 v68, 16, v[vgprValuC+68] // convert C to bf16 +buffer_store_short v68, v65, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v17, 1.0, v17, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+72], v17, v[vgprValuC+72] // *= scaleAlphaVecVMul +v_add_f32 v4, v16, v[vgprValuC+72] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v72, 16, v[vgprValuC+72] // convert C to bf16 +buffer_store_short v72, v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v23, 1.0, v23, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+76], v23, v[vgprValuC+76] // *= scaleAlphaVecVMul +v_add_f32 v4, v22, v[vgprValuC+76] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v76, 16, v[vgprValuC+76] // convert C to bf16 +buffer_store_short v76, v73, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v29, 1.0, v29, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v29, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_add_f32 v4, v28, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v77, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v35, 1.0, v35, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+84], v35, v[vgprValuC+84] // *= scaleAlphaVecVMul +v_add_f32 v4, v34, v[vgprValuC+84] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v84, 16, v[vgprValuC+84] // convert C to bf16 +buffer_store_short v84, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +// jump to end +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_End_2, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_End_2 +label_GW_Beta_2: +s_and_b32 s60, 255, s[sgprSizeI] // s60 = s[sgprSizeI] % 256 +s_add_u32 s61, -0x1, s[sgprNumWorkGroups0] +s_cmp_ge_u32 s[sgprWorkGroup0], s61 // wg0 >= nwg0-1 ? +s_cselect_b32 s60, s60, 0 // set rMT0 +s_cmpk_gt_u32 s60, 0x0 // rMT0 > 0 +s_cbranch_scc0 label_NoBranch_1L38YJQL3BUJ48XK_0 // Only branch on scc1 +// jump if edges required +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_B1_E1_M, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_B1_E1_M +label_NoBranch_1L38YJQL3BUJ48XK_0: +s_and_b32 s60, 255, s[sgprSizeJ] // s60 = s[sgprSizeJ] % 256 +s_add_u32 s61, -0x1, s[sgprNumWorkGroups1] +s_cmp_ge_u32 s[sgprWorkGroup1], s61 // wg1 >= nwg1-1 +s_cselect_b32 s60, s60, 0 // set rMT1 +s_cmpk_gt_u32 s60, 0x0 // rMT1 > 0 +s_cbranch_scc0 label_NoBranch_XMVL70A9XU3BIJFQ_0 // Only branch on scc1 +// jump if edges required +s_getpc_b64 s[60:61] // addr of next instr +s_add_i32 s62, label_GW_B1_E1_N, 0x4 // target branch offset +s_add_u32 s60, s60, s62 // add target branch offset +s_addc_u32 s61, s61, 0 // add high and carry +s_setpc_b64 s[60:61] // branch to label_GW_B1_E1_N +label_NoBranch_XMVL70A9XU3BIJFQ_0: +label_GW_B1_E0: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_1_edge_0 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_1_edge_0 // Branch if true +label_To_Activation_None_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Abs_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Gelu_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Relu_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Sigmoid_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Tanh_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Geluscaling_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_To_Activation_Silu_VW4_1_beta_1_edge_0: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_3 +label_ActivationSetPCAddrEnd_3: + +/* edge=0, allocate 2 sgpr. perBatchTmpS=2 perBatchMaskS=0 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4); (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_add_lshl_u32 v14, v2, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 +buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b128 v[20:23], v15 offset:0 // load bias +v_lshlrev_b32 v16, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +v_add_lshl_u32 v13, v3, v0, 0x1 // optSingleColVgpr scaleToBpe: sharedAddrVgpr <- cinRowPtr + coord0, scaled by BPE. BSHERE:coord0=0, coord0Vgpr=0 +v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+36], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+37], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+38], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+39], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+40], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+41], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+42], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+43], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+48], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+49], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+50], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+51], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+52], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+53], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+54], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+55], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+60], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+61], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+62], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+63], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+64], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+65], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+66], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+67], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+72], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+73], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+74], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+75], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+76], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+77], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+78], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+79], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+84], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+85], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+86], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+87], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+88], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+89], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+90], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+91], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+96], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+97], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+98], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+99], acc47 // copy acc to vreg[47] +v_accvgpr_read_b32 v[vgprValuC+100], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+101], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+102], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+103], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+108], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+109], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+110], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+111], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+112], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+113], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+114], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+115], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+120], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+121], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+122], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+123], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0), (3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha +v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha +v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha +v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha +v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha +v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha +v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha +v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v18, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v19, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v32, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v33, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v44, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v45, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v46, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v47, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v56, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v57, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v58, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v59, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v68, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v69, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v80, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v81, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v82, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v83, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v92, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v93, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v94, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v95, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v104, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v105, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v112, v4 +v_mov_b32 v113, v5 +v_mov_b32 v114, v6 +v_mov_b32 v115, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan +v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan +v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] +v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan +v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan +v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] +v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v116, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v117, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v120, v4 +v_mov_b32 v121, v5 +v_mov_b32 v122, v6 +v_mov_b32 v123, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan +v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan +v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] +v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan +v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan +v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] +v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Batch #1 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4); (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +v_accvgpr_read_b32 v[vgprValuC+28], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+29], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+30], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+31], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+36], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+37], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+38], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+39], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+40], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+41], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+42], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+43], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+48], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+49], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+50], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+51], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+52], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+53], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+54], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+55], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+60], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+61], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+62], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+63], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+64], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+65], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+66], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+67], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+72], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+73], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+74], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+75], acc95 // copy acc to vreg[95] +v_accvgpr_read_b32 v[vgprValuC+76], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+77], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+78], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+79], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+84], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+85], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+86], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+87], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+88], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+89], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+90], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+91], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+96], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+97], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+98], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+99], acc111 // copy acc to vreg[111] +v_accvgpr_read_b32 v[vgprValuC+100], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+101], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+102], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+103], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+108], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+109], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+110], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+111], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+112], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+113], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+114], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+115], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+120], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+121], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+122], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+123], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0), (6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha +v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha +v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha +v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha +v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha +v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha +v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha +v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v18, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v19, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v32, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v33, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v44, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v45, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v46, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v47, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v56, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v57, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v58, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v59, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v68, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v69, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v80, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v81, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v82, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v83, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v92, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v93, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v94, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v95, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v104, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v105, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v112, v4 +v_mov_b32 v113, v5 +v_mov_b32 v114, v6 +v_mov_b32 v115, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan +v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan +v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] +v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan +v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan +v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] +v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v116, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v117, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v120, v4 +v_mov_b32 v121, v5 +v_mov_b32 v122, v6 +v_mov_b32 v123, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan +v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan +v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] +v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan +v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan +v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] +v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Batch #2 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4); (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +v_accvgpr_read_b32 v[vgprValuC+28], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+29], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+30], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+31], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+36], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+37], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+38], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+39], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+40], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+41], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+42], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+43], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+48], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+49], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+50], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+51], acc143 // copy acc to vreg[143] +v_accvgpr_read_b32 v[vgprValuC+52], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+53], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+54], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+55], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+60], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+61], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+62], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+63], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+64], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+65], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+66], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+67], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+72], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+73], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+74], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+75], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+76], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+77], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+78], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+79], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+84], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+85], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+86], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+87], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+88], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+89], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+90], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+91], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+96], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+97], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+98], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+99], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+100], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+101], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+102], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+103], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+108], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+109], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+110], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+111], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+112], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+113], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+114], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+115], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+120], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+121], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+122], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+123], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0), (9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha +v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha +v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha +v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha +v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha +v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha +v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha +v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v18, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v19, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v32, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v33, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v44, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v45, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v46, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v47, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v56, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v57, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v58, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v59, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v68, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v69, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v80, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v81, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v82, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v83, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v92, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v93, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v94, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v95, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v104, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v105, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v112, v4 +v_mov_b32 v113, v5 +v_mov_b32 v114, v6 +v_mov_b32 v115, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan +v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan +v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] +v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan +v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan +v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] +v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v116, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v117, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v120, v4 +v_mov_b32 v121, v5 +v_mov_b32 v122, v6 +v_mov_b32 v123, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan +v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan +v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] +v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan +v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan +v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] +v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=1 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Mask optSrdIncForRow=1 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Batch #3 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4); (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[18:19], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v15, v0, s60 +v_lshlrev_b32 v15, 0x2, v15 // Bias address scaled by BPE +ds_read_b128 v[20:23], v15 offset:0 // load bias +buffer_load_dwordx4 v[24:27], v16, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[32:33], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[34:35], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[44:45], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[46:47], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[56:57], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[58:59], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[68:69], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[70:71], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[80:81], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[82:83], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[92:93], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +s_mul_i32 s60, s[sgprStrideC1J], 26 // scale StrideC *= numRows(13) * bpe +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[94:95], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[104:105], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[106:107], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +s_lshl_b32 s60, s[sgprStrideC1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdC+0], s[sgprSrdC+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdC+1], s[sgprSrdC+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_load_dwordx2 v[116:117], v14, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+36], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+37], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+38], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+39], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+40], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+41], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+42], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+43], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+48], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+49], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+50], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+51], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+52], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+53], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+54], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+55], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+60], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+61], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+62], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+63], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+64], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+65], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+66], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+67], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+72], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+73], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+74], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+75], acc223 // copy acc to vreg[223] +v_accvgpr_read_b32 v[vgprValuC+76], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+77], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+78], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+79], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+84], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+85], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+86], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+87], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+88], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+89], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+90], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+91], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+96], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+97], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+98], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+99], acc239 // copy acc to vreg[239] +v_accvgpr_read_b32 v[vgprValuC+100], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+101], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+102], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+103], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+108], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+109], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+110], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+111], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+112], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+113], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+114], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+115], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+120], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+121], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+122], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+123], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0), (15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+41], s[sgprAlpha], v[vgprValuC+41] // *= alpha +v_mul_f32 v[vgprValuC+42], s[sgprAlpha], v[vgprValuC+42] // *= alpha +v_mul_f32 v[vgprValuC+43], s[sgprAlpha], v[vgprValuC+43] // *= alpha +v_mul_f32 v[vgprValuC+48], s[sgprAlpha], v[vgprValuC+48] // *= alpha +v_mul_f32 v[vgprValuC+49], s[sgprAlpha], v[vgprValuC+49] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+51], s[sgprAlpha], v[vgprValuC+51] // *= alpha +v_mul_f32 v[vgprValuC+52], s[sgprAlpha], v[vgprValuC+52] // *= alpha +v_mul_f32 v[vgprValuC+53], s[sgprAlpha], v[vgprValuC+53] // *= alpha +v_mul_f32 v[vgprValuC+54], s[sgprAlpha], v[vgprValuC+54] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+61], s[sgprAlpha], v[vgprValuC+61] // *= alpha +v_mul_f32 v[vgprValuC+62], s[sgprAlpha], v[vgprValuC+62] // *= alpha +v_mul_f32 v[vgprValuC+63], s[sgprAlpha], v[vgprValuC+63] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+76], s[sgprAlpha], v[vgprValuC+76] // *= alpha +v_mul_f32 v[vgprValuC+77], s[sgprAlpha], v[vgprValuC+77] // *= alpha +v_mul_f32 v[vgprValuC+78], s[sgprAlpha], v[vgprValuC+78] // *= alpha +v_mul_f32 v[vgprValuC+79], s[sgprAlpha], v[vgprValuC+79] // *= alpha +v_mul_f32 v[vgprValuC+84], s[sgprAlpha], v[vgprValuC+84] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+86], s[sgprAlpha], v[vgprValuC+86] // *= alpha +v_mul_f32 v[vgprValuC+87], s[sgprAlpha], v[vgprValuC+87] // *= alpha +v_mul_f32 v[vgprValuC+88], s[sgprAlpha], v[vgprValuC+88] // *= alpha +v_mul_f32 v[vgprValuC+89], s[sgprAlpha], v[vgprValuC+89] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+91], s[sgprAlpha], v[vgprValuC+91] // *= alpha +v_mul_f32 v[vgprValuC+96], s[sgprAlpha], v[vgprValuC+96] // *= alpha +v_mul_f32 v[vgprValuC+97], s[sgprAlpha], v[vgprValuC+97] // *= alpha +v_mul_f32 v[vgprValuC+98], s[sgprAlpha], v[vgprValuC+98] // *= alpha +v_mul_f32 v[vgprValuC+99], s[sgprAlpha], v[vgprValuC+99] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+112], s[sgprAlpha], v[vgprValuC+112] // *= alpha +v_mul_f32 v[vgprValuC+113], s[sgprAlpha], v[vgprValuC+113] // *= alpha +v_mul_f32 v[vgprValuC+114], s[sgprAlpha], v[vgprValuC+114] // *= alpha +v_mul_f32 v[vgprValuC+115], s[sgprAlpha], v[vgprValuC+115] // *= alpha +v_mul_f32 v[vgprValuC+120], s[sgprAlpha], v[vgprValuC+120] // *= alpha +v_mul_f32 v[vgprValuC+121], s[sgprAlpha], v[vgprValuC+121] // *= alpha +v_mul_f32 v[vgprValuC+122], s[sgprAlpha], v[vgprValuC+122] // *= alpha +v_mul_f32 v[vgprValuC+123], s[sgprAlpha], v[vgprValuC+123] // *= alpha + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 + +s_waitcnt lgkmcnt(0), vmcnt(15) // vmcnt(15) = 17 - 1 (beta) - 1 (scaleAlphaVec) lgkmcnt(0) = 1 - 1 (bias) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v18 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v18, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v19 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v19, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(14) = 17 - 2 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v32 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v32, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v33 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v33, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[36:37], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(13) = 17 - 3 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+40:vgprValuC+40+1], v[24:25], v[vgprValuC+40:vgprValuC+40+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+42:vgprValuC+42+1], v[26:27], v[vgprValuC+42:vgprValuC+42+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+41], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+42], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+43], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+40:vgprValuC+40+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+42:vgprValuC+42+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_mov_b32 v41, v5 +v_mov_b32 v42, v6 +v_mov_b32 v43, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+40], 16, v[vgprValuC+40] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+41], v[vgprValuC+41] // check Nan +v_bfe_u32 v9, v[vgprValuC+41], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+41], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+41], v9, v11, s[60:61] +v_and_or_b32 v40, v[vgprValuC+41], v10, v[vgprValuC+40] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+42], v[vgprValuC+42] // check Nan +v_bfe_u32 v9, v[vgprValuC+42], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+42], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+42], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+42], 16, v[vgprValuC+42] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+43], v[vgprValuC+43] // check Nan +v_bfe_u32 v9, v[vgprValuC+43], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+43], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+43], v9, v11, s[60:61] +v_and_or_b32 v41, v[vgprValuC+43], v10, v[vgprValuC+42] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[40:41], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(12) = 17 - 4 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+48:vgprValuC+48+1], v[24:25], v[vgprValuC+48:vgprValuC+48+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+50:vgprValuC+50+1], v[26:27], v[vgprValuC+50:vgprValuC+50+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+48], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v44, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+49], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v45 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v45, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+51], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+48:vgprValuC+48+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+50:vgprValuC+50+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v48, v4 +v_mov_b32 v49, v5 +v_mov_b32 v50, v6 +v_mov_b32 v51, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+48], v[vgprValuC+48] // check Nan +v_bfe_u32 v9, v[vgprValuC+48], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+48], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+48], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+48], 16, v[vgprValuC+48] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+49], v[vgprValuC+49] // check Nan +v_bfe_u32 v9, v[vgprValuC+49], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+49], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+49], v9, v11, s[60:61] +v_and_or_b32 v48, v[vgprValuC+49], v10, v[vgprValuC+48] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+50], 16, v[vgprValuC+50] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+51], v[vgprValuC+51] // check Nan +v_bfe_u32 v9, v[vgprValuC+51], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+51], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+51], v9, v11, s[60:61] +v_and_or_b32 v49, v[vgprValuC+51], v10, v[vgprValuC+50] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[48:49], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(11) = 17 - 5 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+52:vgprValuC+52+1], v[24:25], v[vgprValuC+52:vgprValuC+52+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+54:vgprValuC+54+1], v[26:27], v[vgprValuC+54:vgprValuC+54+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v46 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+52], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v46, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+53], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v47 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+54], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v47, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+52:vgprValuC+52+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+54:vgprValuC+54+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v52, v4 +v_mov_b32 v53, v5 +v_mov_b32 v54, v6 +v_mov_b32 v55, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+52], v[vgprValuC+52] // check Nan +v_bfe_u32 v9, v[vgprValuC+52], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+52], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+52], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+52], 16, v[vgprValuC+52] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+53], v[vgprValuC+53] // check Nan +v_bfe_u32 v9, v[vgprValuC+53], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+53], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+53], v9, v11, s[60:61] +v_and_or_b32 v52, v[vgprValuC+53], v10, v[vgprValuC+52] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+54], v[vgprValuC+54] // check Nan +v_bfe_u32 v9, v[vgprValuC+54], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+54], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+54], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+54], 16, v[vgprValuC+54] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_and_or_b32 v53, v[vgprValuC+55], v10, v[vgprValuC+54] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[52:53], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(10) = 17 - 6 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+60:vgprValuC+60+1], v[24:25], v[vgprValuC+60:vgprValuC+60+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+62:vgprValuC+62+1], v[26:27], v[vgprValuC+62:vgprValuC+62+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v56 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v56, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+61], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v57 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+62], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v57, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+63], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+60:vgprValuC+60+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+62:vgprValuC+62+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_mov_b32 v61, v5 +v_mov_b32 v62, v6 +v_mov_b32 v63, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+60], 16, v[vgprValuC+60] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+61], v[vgprValuC+61] // check Nan +v_bfe_u32 v9, v[vgprValuC+61], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+61], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+61], v9, v11, s[60:61] +v_and_or_b32 v60, v[vgprValuC+61], v10, v[vgprValuC+60] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+62], v[vgprValuC+62] // check Nan +v_bfe_u32 v9, v[vgprValuC+62], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+62], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+62], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+62], 16, v[vgprValuC+62] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+63], v[vgprValuC+63] // check Nan +v_bfe_u32 v9, v[vgprValuC+63], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+63], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+63], v9, v11, s[60:61] +v_and_or_b32 v61, v[vgprValuC+63], v10, v[vgprValuC+62] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[60:61], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(9) = 17 - 7 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v58 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v58, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v59, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[64:65], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(8) = 17 - 8 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v68 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v68, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v69, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[72:73], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(7) = 17 - 9 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+76:vgprValuC+76+1], v[24:25], v[vgprValuC+76:vgprValuC+76+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+78:vgprValuC+78+1], v[26:27], v[vgprValuC+78:vgprValuC+78+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+76], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+77], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+78], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+79], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+76:vgprValuC+76+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+78:vgprValuC+78+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v76, v4 +v_mov_b32 v77, v5 +v_mov_b32 v78, v6 +v_mov_b32 v79, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+76], v[vgprValuC+76] // check Nan +v_bfe_u32 v9, v[vgprValuC+76], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+76], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+76], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+76], 16, v[vgprValuC+76] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+77], v[vgprValuC+77] // check Nan +v_bfe_u32 v9, v[vgprValuC+77], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+77], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+77], v9, v11, s[60:61] +v_and_or_b32 v76, v[vgprValuC+77], v10, v[vgprValuC+76] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+78], v[vgprValuC+78] // check Nan +v_bfe_u32 v9, v[vgprValuC+78], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+78], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+78], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+78], 16, v[vgprValuC+78] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+79], v[vgprValuC+79] // check Nan +v_bfe_u32 v9, v[vgprValuC+79], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+79], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+79], v9, v11, s[60:61] +v_and_or_b32 v77, v[vgprValuC+79], v10, v[vgprValuC+78] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[76:77], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(6) = 17 - 10 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+84:vgprValuC+84+1], v[24:25], v[vgprValuC+84:vgprValuC+84+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+86:vgprValuC+86+1], v[26:27], v[vgprValuC+86:vgprValuC+86+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v80 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+84], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v80, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v81 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+86], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v81, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+87], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+84:vgprValuC+84+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+86:vgprValuC+86+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v84, v4 +v_mov_b32 v85, v5 +v_mov_b32 v86, v6 +v_mov_b32 v87, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+84], v[vgprValuC+84] // check Nan +v_bfe_u32 v9, v[vgprValuC+84], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+84], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+84], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+84], 16, v[vgprValuC+84] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_and_or_b32 v84, v[vgprValuC+85], v10, v[vgprValuC+84] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+86], v[vgprValuC+86] // check Nan +v_bfe_u32 v9, v[vgprValuC+86], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+86], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+86], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+86], 16, v[vgprValuC+86] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+87], v[vgprValuC+87] // check Nan +v_bfe_u32 v9, v[vgprValuC+87], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+87], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+87], v9, v11, s[60:61] +v_and_or_b32 v85, v[vgprValuC+87], v10, v[vgprValuC+86] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[84:85], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(5) = 17 - 11 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+88:vgprValuC+88+1], v[24:25], v[vgprValuC+88:vgprValuC+88+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+90:vgprValuC+90+1], v[26:27], v[vgprValuC+90:vgprValuC+90+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v82 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+88], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v82, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+89], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v83 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v83, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+91], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+88:vgprValuC+88+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+90:vgprValuC+90+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v88, v4 +v_mov_b32 v89, v5 +v_mov_b32 v90, v6 +v_mov_b32 v91, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+88], v[vgprValuC+88] // check Nan +v_bfe_u32 v9, v[vgprValuC+88], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+88], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+88], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+88], 16, v[vgprValuC+88] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+89], v[vgprValuC+89] // check Nan +v_bfe_u32 v9, v[vgprValuC+89], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+89], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+89], v9, v11, s[60:61] +v_and_or_b32 v88, v[vgprValuC+89], v10, v[vgprValuC+88] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+90], 16, v[vgprValuC+90] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+91], v[vgprValuC+91] // check Nan +v_bfe_u32 v9, v[vgprValuC+91], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+91], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+91], v9, v11, s[60:61] +v_and_or_b32 v89, v[vgprValuC+91], v10, v[vgprValuC+90] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[88:89], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(4) = 17 - 12 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+96:vgprValuC+96+1], v[24:25], v[vgprValuC+96:vgprValuC+96+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+98:vgprValuC+98+1], v[26:27], v[vgprValuC+98:vgprValuC+98+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v92 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+96], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v92, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+97], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v93 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+98], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v93, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+99], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+96:vgprValuC+96+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+98:vgprValuC+98+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v96, v4 +v_mov_b32 v97, v5 +v_mov_b32 v98, v6 +v_mov_b32 v99, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+96], v[vgprValuC+96] // check Nan +v_bfe_u32 v9, v[vgprValuC+96], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+96], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+96], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+96], 16, v[vgprValuC+96] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+97], v[vgprValuC+97] // check Nan +v_bfe_u32 v9, v[vgprValuC+97], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+97], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+97], v9, v11, s[60:61] +v_and_or_b32 v96, v[vgprValuC+97], v10, v[vgprValuC+96] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+98], v[vgprValuC+98] // check Nan +v_bfe_u32 v9, v[vgprValuC+98], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+98], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+98], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+98], 16, v[vgprValuC+98] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+99], v[vgprValuC+99] // check Nan +v_bfe_u32 v9, v[vgprValuC+99], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+99], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+99], v9, v11, s[60:61] +v_and_or_b32 v97, v[vgprValuC+99], v10, v[vgprValuC+98] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[96:97], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(3) = 17 - 13 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v94, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v95 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v95, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +s_mul_i32 s60, s[sgprStrideD1J], 26 // scale StrideD *= numRows(13) * bpe +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[100:101], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(2) = 17 - 14 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v104 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v104, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v105 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v105, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[108:109], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(1) = 17 - 15 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+112:vgprValuC+112+1], v[24:25], v[vgprValuC+112:vgprValuC+112+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+114:vgprValuC+114+1], v[26:27], v[vgprValuC+114:vgprValuC+114+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+112], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+113], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+114], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+115], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+112:vgprValuC+112+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+114:vgprValuC+114+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v112, v4 +v_mov_b32 v113, v5 +v_mov_b32 v114, v6 +v_mov_b32 v115, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+112], v[vgprValuC+112] // check Nan +v_bfe_u32 v9, v[vgprValuC+112], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+112], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+112], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+112], 16, v[vgprValuC+112] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+113], v[vgprValuC+113] // check Nan +v_bfe_u32 v9, v[vgprValuC+113], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+113], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+113], v9, v11, s[60:61] +v_and_or_b32 v112, v[vgprValuC+113], v10, v[vgprValuC+112] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+114], v[vgprValuC+114] // check Nan +v_bfe_u32 v9, v[vgprValuC+114], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+114], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+114], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+114], 16, v[vgprValuC+114] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+115], v[vgprValuC+115] // check Nan +v_bfe_u32 v9, v[vgprValuC+115], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+115], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+115], v9, v11, s[60:61] +v_and_or_b32 v113, v[vgprValuC+115], v10, v[vgprValuC+114] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[112:113], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D + +s_waitcnt vmcnt(15) // vmcnt(0) = 17 - 16 (beta) - 1 (scaleAlphaVec) (interleaved) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+120:vgprValuC+120+1], v[24:25], v[vgprValuC+120:vgprValuC+120+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+122:vgprValuC+122+1], v[26:27], v[vgprValuC+122:vgprValuC+122+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v116 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+120], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v116, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+121], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v117 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+122], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v117, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+123], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+120:vgprValuC+120+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+122:vgprValuC+122+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v120, v4 +v_mov_b32 v121, v5 +v_mov_b32 v122, v6 +v_mov_b32 v123, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+120], v[vgprValuC+120] // check Nan +v_bfe_u32 v9, v[vgprValuC+120], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+120], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+120], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+120], 16, v[vgprValuC+120] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+121], v[vgprValuC+121] // check Nan +v_bfe_u32 v9, v[vgprValuC+121], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+121], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+121], v9, v11, s[60:61] +v_and_or_b32 v120, v[vgprValuC+121], v10, v[vgprValuC+120] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+122], v[vgprValuC+122] // check Nan +v_bfe_u32 v9, v[vgprValuC+122], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+122], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+122], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+122], 16, v[vgprValuC+122] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+123], v[vgprValuC+123] // check Nan +v_bfe_u32 v9, v[vgprValuC+123], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+123], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+123], v9, v11, s[60:61] +v_and_or_b32 v121, v[vgprValuC+123], v10, v[vgprValuC+122] // pack two bf16 to dword +s_lshl_b32 s60, s[sgprStrideD1J], 1 // incToNextRow: Scale by BPE +s_add_u32 s[sgprSrdD+0], s[sgprSrdD+0], s60 // incToNextRow: gra SRD += inc(lower) +s_addc_u32 s[sgprSrdD+1], s[sgprSrdD+1], 0 // incToNextRow: gra SRD += inc(upper) +buffer_store_dwordx2 v[120:121], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_2 // jump to end +label_GW_B1_E1_N: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW4_1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW4_1_beta_1_edge_1 // Branch if true +label_To_Activation_None_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Abs_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Clippedrelu_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Gelu_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Leakyrelu_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Relu_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Sigmoid_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Tanh_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Geluscaling_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_To_Activation_Silu_VW4_1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW4, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_2 +label_ActivationSetPCAddrEnd_2: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=12 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw4); (0,0,1,0:vw4); (0,0,2,0:vw4); (0,0,3,0:vw4); (1,0,0,0:vw4); (1,0,1,0:vw4); (1,0,2,0:vw4); (1,0,3,0:vw4); (2,0,0,0:vw4); (2,0,1,0:vw4); (2,0,2,0:vw4); (2,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v123, BufferOOB +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b128 v[20:23], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v19, v0, s60 +v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE +v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v40, v0, s60 +v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE +v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v49, v0, s60 +v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE +v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v63, v0, s60 +v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE +v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v76, v0, s60 +v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE +v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v90, v0, s60 +v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE +v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v99, v0, s60 +v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE +v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v112, v0, s60 +v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE +v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v121, v0, s60 +v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE +v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+28], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+29], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+30], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+31], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+36], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+37], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+38], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+39], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+44], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+45], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+46], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+47], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+56], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+57], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+58], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+59], acc15 // copy acc to vreg[15] +v_accvgpr_read_b32 v[vgprValuC+64], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+65], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+66], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+67], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+72], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+73], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+74], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+75], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+80], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+81], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+82], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+83], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+92], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+93], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+94], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+95], acc31 // copy acc to vreg[31] +v_accvgpr_read_b32 v[vgprValuC+100], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+101], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+102], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+103], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+108], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+109], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+110], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+111], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+116], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+117], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+118], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+119], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+128], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+129], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+130], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+131], acc47 // copy acc to vreg[47] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 1, 0), (0, 0, 2, 0), (0, 0, 3, 0), (1, 0, 0, 0), (1, 0, 1, 0), (1, 0, 2, 0), (1, 0, 3, 0), (2, 0, 0, 0), (2, 0, 1, 0), (2, 0, 2, 0), (2, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha +v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha +v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha +v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha +v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha +v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v16, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v17, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v42, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v43, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v52, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v53, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v60, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v61, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v78, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v79, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v88, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v89, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v92, v4 +v_mov_b32 v93, v5 +v_mov_b32 v94, v6 +v_mov_b32 v95, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan +v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan +v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] +v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan +v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword +buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v96, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v97, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v114, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v115, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v124, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v125, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v128, v4 +v_mov_b32 v129, v5 +v_mov_b32 v130, v6 +v_mov_b32 v131, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan +v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan +v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] +v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan +v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan +v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] +v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword +buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw4); (3,0,1,0:vw4); (3,0,2,0:vw4); (3,0,3,0:vw4); (4,0,0,0:vw4); (4,0,1,0:vw4); (4,0,2,0:vw4); (4,0,3,0:vw4); (5,0,0,0:vw4); (5,0,1,0:vw4); (5,0,2,0:vw4); (5,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v123, BufferOOB +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[20:23], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v19, v0, s60 +v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE +v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v40, v0, s60 +v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE +v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v49, v0, s60 +v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE +v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v63, v0, s60 +v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE +v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v76, v0, s60 +v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE +v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v90, v0, s60 +v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE +v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v99, v0, s60 +v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE +v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v112, v0, s60 +v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE +v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v121, v0, s60 +v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE +v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+28], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+29], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+30], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+31], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+36], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+37], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+38], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+39], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+44], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+45], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+46], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+47], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+56], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+57], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+58], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+59], acc63 // copy acc to vreg[63] +v_accvgpr_read_b32 v[vgprValuC+64], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+65], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+66], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+67], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+72], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+73], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+74], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+75], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+80], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+81], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+82], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+83], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+92], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+93], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+94], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+95], acc79 // copy acc to vreg[79] +v_accvgpr_read_b32 v[vgprValuC+100], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+101], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+102], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+103], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+108], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+109], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+110], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+111], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+116], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+117], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+118], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+119], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+128], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+129], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+130], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+131], acc95 // copy acc to vreg[95] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 1, 0), (3, 0, 2, 0), (3, 0, 3, 0), (4, 0, 0, 0), (4, 0, 1, 0), (4, 0, 2, 0), (4, 0, 3, 0), (5, 0, 0, 0), (5, 0, 1, 0), (5, 0, 2, 0), (5, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha +v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha +v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha +v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha +v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha +v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v16, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v17, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v42, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v43, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v52, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v53, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v60, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v61, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v78, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v79, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v88, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v89, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v92, v4 +v_mov_b32 v93, v5 +v_mov_b32 v94, v6 +v_mov_b32 v95, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan +v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan +v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] +v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan +v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword +buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v96, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v97, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v114, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v115, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v124, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v125, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v128, v4 +v_mov_b32 v129, v5 +v_mov_b32 v130, v6 +v_mov_b32 v131, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan +v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan +v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] +v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan +v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan +v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] +v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword +buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw4); (6,0,1,0:vw4); (6,0,2,0:vw4); (6,0,3,0:vw4); (7,0,0,0:vw4); (7,0,1,0:vw4); (7,0,2,0:vw4); (7,0,3,0:vw4); (8,0,0,0:vw4); (8,0,1,0:vw4); (8,0,2,0:vw4); (8,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v123, BufferOOB +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[20:23], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v19, v0, s60 +v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE +v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v40, v0, s60 +v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE +v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v49, v0, s60 +v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE +v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v63, v0, s60 +v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE +v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v76, v0, s60 +v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE +v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v90, v0, s60 +v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE +v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v99, v0, s60 +v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE +v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v112, v0, s60 +v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE +v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v121, v0, s60 +v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE +v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+28], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+29], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+30], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+31], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+36], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+37], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+38], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+39], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+44], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+45], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+46], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+47], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+56], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+57], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+58], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+59], acc111 // copy acc to vreg[111] +v_accvgpr_read_b32 v[vgprValuC+64], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+65], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+66], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+67], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+72], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+73], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+74], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+75], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+80], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+81], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+82], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+83], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+92], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+93], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+94], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+95], acc127 // copy acc to vreg[127] +v_accvgpr_read_b32 v[vgprValuC+100], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+101], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+102], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+103], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+108], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+109], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+110], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+111], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+116], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+117], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+118], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+119], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+128], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+129], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+130], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+131], acc143 // copy acc to vreg[143] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 1, 0), (6, 0, 2, 0), (6, 0, 3, 0), (7, 0, 0, 0), (7, 0, 1, 0), (7, 0, 2, 0), (7, 0, 3, 0), (8, 0, 0, 0), (8, 0, 1, 0), (8, 0, 2, 0), (8, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha +v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha +v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha +v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha +v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha +v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v16, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v17, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v42, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v43, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v52, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v53, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v60, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v61, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v78, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v79, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v88, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v89, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v92, v4 +v_mov_b32 v93, v5 +v_mov_b32 v94, v6 +v_mov_b32 v95, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan +v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan +v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] +v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan +v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword +buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v96, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v97, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v114, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v115, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v124, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v125, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v128, v4 +v_mov_b32 v129, v5 +v_mov_b32 v130, v6 +v_mov_b32 v131, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan +v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan +v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] +v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan +v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan +v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] +v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword +buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw4); (9,0,1,0:vw4); (9,0,2,0:vw4); (9,0,3,0:vw4); (10,0,0,0:vw4); (10,0,1,0:vw4); (10,0,2,0:vw4); (10,0,3,0:vw4); (11,0,0,0:vw4); (11,0,1,0:vw4); (11,0,2,0:vw4); (11,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v123, BufferOOB +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[20:23], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v19, v0, s60 +v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE +v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v40, v0, s60 +v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE +v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v49, v0, s60 +v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE +v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v63, v0, s60 +v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE +v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v76, v0, s60 +v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE +v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v90, v0, s60 +v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE +v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v99, v0, s60 +v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE +v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v112, v0, s60 +v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE +v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v121, v0, s60 +v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE +v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+28], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+29], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+30], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+31], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+36], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+37], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+38], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+39], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+44], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+45], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+46], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+47], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+56], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+57], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+58], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+59], acc159 // copy acc to vreg[159] +v_accvgpr_read_b32 v[vgprValuC+64], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+65], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+66], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+67], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+72], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+73], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+74], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+75], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+80], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+81], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+82], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+83], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+92], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+93], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+94], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+95], acc175 // copy acc to vreg[175] +v_accvgpr_read_b32 v[vgprValuC+100], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+101], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+102], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+103], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+108], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+109], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+110], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+111], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+116], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+117], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+118], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+119], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+128], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+129], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+130], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+131], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 1, 0), (9, 0, 2, 0), (9, 0, 3, 0), (10, 0, 0, 0), (10, 0, 1, 0), (10, 0, 2, 0), (10, 0, 3, 0), (11, 0, 0, 0), (11, 0, 1, 0), (11, 0, 2, 0), (11, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha +v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha +v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha +v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha +v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha +v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v16, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v17, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v42, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v43, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v52, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v53, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v60, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v61, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v78, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v79, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v88, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v89, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v92, v4 +v_mov_b32 v93, v5 +v_mov_b32 v94, v6 +v_mov_b32 v95, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan +v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan +v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] +v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan +v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword +buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v96, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v97, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v114, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v115, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v124, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v125, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v128, v4 +v_mov_b32 v129, v5 +v_mov_b32 v130, v6 +v_mov_b32 v131, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan +v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan +v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] +v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan +v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan +v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] +v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword +buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw4); (12,0,1,0:vw4); (12,0,2,0:vw4); (12,0,3,0:vw4); (13,0,0,0:vw4); (13,0,1,0:vw4); (13,0,2,0:vw4); (13,0,3,0:vw4); (14,0,0,0:vw4); (14,0,1,0:vw4); (14,0,2,0:vw4); (14,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v123, BufferOOB +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v123, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[20:23], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v123, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v19, v0, s60 +v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE +v_cndmask_b32 v19, v123, v19, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v123, v18, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v40, v0, s60 +v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE +v_cndmask_b32 v40, v123, v40, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v123, v33, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v49, v0, s60 +v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE +v_cndmask_b32 v49, v123, v49, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v123, v48, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[60:61], v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v54, v0, s60 +v_lshlrev_b32 v54, 0x2, v54 // Bias address scaled by BPE +v_cndmask_b32 v54, v123, v54, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v55, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v123, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v62, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[70:71], v62, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v63, v0, s60 +v_lshlrev_b32 v63, 0x2, v63 // Bias address scaled by BPE +v_cndmask_b32 v63, v123, v63, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v62, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v62, v123, v62, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v69, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[78:79], v69, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v76, v0, s60 +v_lshlrev_b32 v76, 0x2, v76 // Bias address scaled by BPE +v_cndmask_b32 v76, v123, v76, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v77, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v69, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v69, v123, v69, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v84, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[88:89], v84, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v85, v0, s60 +v_lshlrev_b32 v85, 0x2, v85 // Bias address scaled by BPE +v_cndmask_b32 v85, v123, v85, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v86, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v84, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v84, v123, v84, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v87, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[96:97], v87, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v90, v0, s60 +v_lshlrev_b32 v90, 0x2, v90 // Bias address scaled by BPE +v_cndmask_b32 v90, v123, v90, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v91, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v87, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v87, v123, v87, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v98, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[106:107], v98, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v99, v0, s60 +v_lshlrev_b32 v99, 0x2, v99 // Bias address scaled by BPE +v_cndmask_b32 v99, v123, v99, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v104, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v98, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v98, v123, v98, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v105, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[114:115], v105, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v112, v0, s60 +v_lshlrev_b32 v112, 0x2, v112 // Bias address scaled by BPE +v_cndmask_b32 v112, v123, v112, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v113, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v105, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v105, v123, v105, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v120, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[124:125], v120, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v121, v0, s60 +v_lshlrev_b32 v121, 0x2, v121 // Bias address scaled by BPE +v_cndmask_b32 v121, v123, v121, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v122, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v120, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v120, v123, v120, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+28], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+29], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+30], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+31], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+36], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+37], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+38], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+39], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+44], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+45], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+46], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+47], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+56], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+57], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+58], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+59], acc207 // copy acc to vreg[207] +v_accvgpr_read_b32 v[vgprValuC+64], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+65], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+66], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+67], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+72], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+73], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+74], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+75], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+80], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+81], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+82], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+83], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+92], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+93], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+94], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+95], acc223 // copy acc to vreg[223] +v_accvgpr_read_b32 v[vgprValuC+100], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+101], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+102], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+103], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+108], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+109], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+110], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+111], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+116], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+117], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+118], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+119], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+128], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+129], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+130], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+131], acc239 // copy acc to vreg[239] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 1, 0), (12, 0, 2, 0), (12, 0, 3, 0), (13, 0, 0, 0), (13, 0, 1, 0), (13, 0, 2, 0), (13, 0, 3, 0), (14, 0, 0, 0), (14, 0, 1, 0), (14, 0, 2, 0), (14, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +v_mul_f32 v[vgprValuC+64], s[sgprAlpha], v[vgprValuC+64] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+66], s[sgprAlpha], v[vgprValuC+66] // *= alpha +v_mul_f32 v[vgprValuC+67], s[sgprAlpha], v[vgprValuC+67] // *= alpha +v_mul_f32 v[vgprValuC+72], s[sgprAlpha], v[vgprValuC+72] // *= alpha +v_mul_f32 v[vgprValuC+73], s[sgprAlpha], v[vgprValuC+73] // *= alpha +v_mul_f32 v[vgprValuC+74], s[sgprAlpha], v[vgprValuC+74] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+81], s[sgprAlpha], v[vgprValuC+81] // *= alpha +v_mul_f32 v[vgprValuC+82], s[sgprAlpha], v[vgprValuC+82] // *= alpha +v_mul_f32 v[vgprValuC+83], s[sgprAlpha], v[vgprValuC+83] // *= alpha +v_mul_f32 v[vgprValuC+92], s[sgprAlpha], v[vgprValuC+92] // *= alpha +v_mul_f32 v[vgprValuC+93], s[sgprAlpha], v[vgprValuC+93] // *= alpha +v_mul_f32 v[vgprValuC+94], s[sgprAlpha], v[vgprValuC+94] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +v_mul_f32 v[vgprValuC+101], s[sgprAlpha], v[vgprValuC+101] // *= alpha +v_mul_f32 v[vgprValuC+102], s[sgprAlpha], v[vgprValuC+102] // *= alpha +v_mul_f32 v[vgprValuC+103], s[sgprAlpha], v[vgprValuC+103] // *= alpha +v_mul_f32 v[vgprValuC+108], s[sgprAlpha], v[vgprValuC+108] // *= alpha +v_mul_f32 v[vgprValuC+109], s[sgprAlpha], v[vgprValuC+109] // *= alpha +v_mul_f32 v[vgprValuC+110], s[sgprAlpha], v[vgprValuC+110] // *= alpha +v_mul_f32 v[vgprValuC+111], s[sgprAlpha], v[vgprValuC+111] // *= alpha +v_mul_f32 v[vgprValuC+116], s[sgprAlpha], v[vgprValuC+116] // *= alpha +v_mul_f32 v[vgprValuC+117], s[sgprAlpha], v[vgprValuC+117] // *= alpha +v_mul_f32 v[vgprValuC+118], s[sgprAlpha], v[vgprValuC+118] // *= alpha +v_mul_f32 v[vgprValuC+119], s[sgprAlpha], v[vgprValuC+119] // *= alpha +v_mul_f32 v[vgprValuC+128], s[sgprAlpha], v[vgprValuC+128] // *= alpha +v_mul_f32 v[vgprValuC+129], s[sgprAlpha], v[vgprValuC+129] // *= alpha +v_mul_f32 v[vgprValuC+130], s[sgprAlpha], v[vgprValuC+130] // *= alpha +v_mul_f32 v[vgprValuC+131], s[sgprAlpha], v[vgprValuC+131] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v16, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v17, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v42, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v43, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v52, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v53, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+64:vgprValuC+64+1], v[24:25], v[vgprValuC+64:vgprValuC+64+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+66:vgprValuC+66+1], v[26:27], v[vgprValuC+66:vgprValuC+66+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v60 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+64], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v60, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v61 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+66], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v61, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+67], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+64:vgprValuC+64+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+66:vgprValuC+66+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v64, v4 +v_mov_b32 v65, v5 +v_mov_b32 v66, v6 +v_mov_b32 v67, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+64], v[vgprValuC+64] // check Nan +v_bfe_u32 v9, v[vgprValuC+64], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+64], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+64], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+64], 16, v[vgprValuC+64] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_and_or_b32 v64, v[vgprValuC+65], v10, v[vgprValuC+64] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+66], v[vgprValuC+66] // check Nan +v_bfe_u32 v9, v[vgprValuC+66], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+66], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+66], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+66], 16, v[vgprValuC+66] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+67], v[vgprValuC+67] // check Nan +v_bfe_u32 v9, v[vgprValuC+67], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+67], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+67], v9, v11, s[60:61] +v_and_or_b32 v65, v[vgprValuC+67], v10, v[vgprValuC+66] // pack two bf16 to dword +buffer_store_dwordx2 v[64:65], v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+72:vgprValuC+72+1], v[24:25], v[vgprValuC+72:vgprValuC+72+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+74:vgprValuC+74+1], v[26:27], v[vgprValuC+74:vgprValuC+74+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v70 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+72], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v70, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+73], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v71 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+74], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v71, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+72:vgprValuC+72+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+74:vgprValuC+74+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v72, v4 +v_mov_b32 v73, v5 +v_mov_b32 v74, v6 +v_mov_b32 v75, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+72], v[vgprValuC+72] // check Nan +v_bfe_u32 v9, v[vgprValuC+72], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+72], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+72], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+72], 16, v[vgprValuC+72] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+73], v[vgprValuC+73] // check Nan +v_bfe_u32 v9, v[vgprValuC+73], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+73], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+73], v9, v11, s[60:61] +v_and_or_b32 v72, v[vgprValuC+73], v10, v[vgprValuC+72] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+74], v[vgprValuC+74] // check Nan +v_bfe_u32 v9, v[vgprValuC+74], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+74], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+74], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+74], 16, v[vgprValuC+74] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_and_or_b32 v73, v[vgprValuC+75], v10, v[vgprValuC+74] // pack two bf16 to dword +buffer_store_dwordx2 v[72:73], v62, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+80:vgprValuC+80+1], v[24:25], v[vgprValuC+80:vgprValuC+80+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+82:vgprValuC+82+1], v[26:27], v[vgprValuC+82:vgprValuC+82+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v78 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v78, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+81], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+82], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v79, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+83], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+80:vgprValuC+80+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+82:vgprValuC+82+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_mov_b32 v81, v5 +v_mov_b32 v82, v6 +v_mov_b32 v83, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+80], 16, v[vgprValuC+80] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+81], v[vgprValuC+81] // check Nan +v_bfe_u32 v9, v[vgprValuC+81], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+81], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+81], v9, v11, s[60:61] +v_and_or_b32 v80, v[vgprValuC+81], v10, v[vgprValuC+80] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+82], v[vgprValuC+82] // check Nan +v_bfe_u32 v9, v[vgprValuC+82], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+82], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+82], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+82], 16, v[vgprValuC+82] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+83], v[vgprValuC+83] // check Nan +v_bfe_u32 v9, v[vgprValuC+83], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+83], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+83], v9, v11, s[60:61] +v_and_or_b32 v81, v[vgprValuC+83], v10, v[vgprValuC+82] // pack two bf16 to dword +buffer_store_dwordx2 v[80:81], v69, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+92:vgprValuC+92+1], v[24:25], v[vgprValuC+92:vgprValuC+92+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+94:vgprValuC+94+1], v[26:27], v[vgprValuC+94:vgprValuC+94+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v88 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+92], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v88, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+93], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+94], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v89, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+92:vgprValuC+92+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+94:vgprValuC+94+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v92, v4 +v_mov_b32 v93, v5 +v_mov_b32 v94, v6 +v_mov_b32 v95, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+92], v[vgprValuC+92] // check Nan +v_bfe_u32 v9, v[vgprValuC+92], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+92], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+92], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+92], 16, v[vgprValuC+92] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+93], v[vgprValuC+93] // check Nan +v_bfe_u32 v9, v[vgprValuC+93], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+93], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+93], v9, v11, s[60:61] +v_and_or_b32 v92, v[vgprValuC+93], v10, v[vgprValuC+92] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+94], v[vgprValuC+94] // check Nan +v_bfe_u32 v9, v[vgprValuC+94], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+94], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+94], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+94], 16, v[vgprValuC+94] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_and_or_b32 v93, v[vgprValuC+95], v10, v[vgprValuC+94] // pack two bf16 to dword +buffer_store_dwordx2 v[92:93], v84, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+100:vgprValuC+100+1], v[24:25], v[vgprValuC+100:vgprValuC+100+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+102:vgprValuC+102+1], v[26:27], v[vgprValuC+102:vgprValuC+102+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v96 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v96, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+101], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v97 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+102], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v97, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+103], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+100:vgprValuC+100+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+102:vgprValuC+102+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_mov_b32 v101, v5 +v_mov_b32 v102, v6 +v_mov_b32 v103, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+100], 16, v[vgprValuC+100] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+101], v[vgprValuC+101] // check Nan +v_bfe_u32 v9, v[vgprValuC+101], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+101], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+101], v9, v11, s[60:61] +v_and_or_b32 v100, v[vgprValuC+101], v10, v[vgprValuC+100] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+102], v[vgprValuC+102] // check Nan +v_bfe_u32 v9, v[vgprValuC+102], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+102], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+102], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+102], 16, v[vgprValuC+102] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+103], v[vgprValuC+103] // check Nan +v_bfe_u32 v9, v[vgprValuC+103], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+103], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+103], v9, v11, s[60:61] +v_and_or_b32 v101, v[vgprValuC+103], v10, v[vgprValuC+102] // pack two bf16 to dword +buffer_store_dwordx2 v[100:101], v87, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+108:vgprValuC+108+1], v[24:25], v[vgprValuC+108:vgprValuC+108+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+110:vgprValuC+110+1], v[26:27], v[vgprValuC+110:vgprValuC+110+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v106 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+108], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v106, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+109], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v107 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+110], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v107, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+111], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+108:vgprValuC+108+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+110:vgprValuC+110+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v108, v4 +v_mov_b32 v109, v5 +v_mov_b32 v110, v6 +v_mov_b32 v111, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+108], v[vgprValuC+108] // check Nan +v_bfe_u32 v9, v[vgprValuC+108], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+108], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+108], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+108], 16, v[vgprValuC+108] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+109], v[vgprValuC+109] // check Nan +v_bfe_u32 v9, v[vgprValuC+109], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+109], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+109], v9, v11, s[60:61] +v_and_or_b32 v108, v[vgprValuC+109], v10, v[vgprValuC+108] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+110], v[vgprValuC+110] // check Nan +v_bfe_u32 v9, v[vgprValuC+110], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+110], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+110], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+110], 16, v[vgprValuC+110] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+111], v[vgprValuC+111] // check Nan +v_bfe_u32 v9, v[vgprValuC+111], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+111], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+111], v9, v11, s[60:61] +v_and_or_b32 v109, v[vgprValuC+111], v10, v[vgprValuC+110] // pack two bf16 to dword +buffer_store_dwordx2 v[108:109], v98, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+116:vgprValuC+116+1], v[24:25], v[vgprValuC+116:vgprValuC+116+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+118:vgprValuC+118+1], v[26:27], v[vgprValuC+118:vgprValuC+118+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v114 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+116], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v114, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+117], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v115 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+118], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v115, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+119], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+116:vgprValuC+116+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+118:vgprValuC+118+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v116, v4 +v_mov_b32 v117, v5 +v_mov_b32 v118, v6 +v_mov_b32 v119, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+116], v[vgprValuC+116] // check Nan +v_bfe_u32 v9, v[vgprValuC+116], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+116], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+116], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+116], 16, v[vgprValuC+116] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+117], v[vgprValuC+117] // check Nan +v_bfe_u32 v9, v[vgprValuC+117], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+117], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+117], v9, v11, s[60:61] +v_and_or_b32 v116, v[vgprValuC+117], v10, v[vgprValuC+116] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+118], v[vgprValuC+118] // check Nan +v_bfe_u32 v9, v[vgprValuC+118], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+118], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+118], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+118], 16, v[vgprValuC+118] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+119], v[vgprValuC+119] // check Nan +v_bfe_u32 v9, v[vgprValuC+119], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+119], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+119], v9, v11, s[60:61] +v_and_or_b32 v117, v[vgprValuC+119], v10, v[vgprValuC+118] // pack two bf16 to dword +buffer_store_dwordx2 v[116:117], v105, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+128:vgprValuC+128+1], v[24:25], v[vgprValuC+128:vgprValuC+128+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+130:vgprValuC+130+1], v[26:27], v[vgprValuC+130:vgprValuC+130+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v124 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+128], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v124, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+129], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v125 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+130], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v125, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+131], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+128:vgprValuC+128+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+130:vgprValuC+130+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v128, v4 +v_mov_b32 v129, v5 +v_mov_b32 v130, v6 +v_mov_b32 v131, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+128], v[vgprValuC+128] // check Nan +v_bfe_u32 v9, v[vgprValuC+128], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+128], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+128], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+128], 16, v[vgprValuC+128] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+129], v[vgprValuC+129] // check Nan +v_bfe_u32 v9, v[vgprValuC+129], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+129], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+129], v9, v11, s[60:61] +v_and_or_b32 v128, v[vgprValuC+129], v10, v[vgprValuC+128] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+130], v[vgprValuC+130] // check Nan +v_bfe_u32 v9, v[vgprValuC+130], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+130], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+130], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+130], 16, v[vgprValuC+130] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+131], v[vgprValuC+131] // check Nan +v_bfe_u32 v9, v[vgprValuC+131], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+131], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+131], v9, v11, s[60:61] +v_and_or_b32 v129, v[vgprValuC+131], v10, v[vgprValuC+130] // pack two bf16 to dword +buffer_store_dwordx2 v[128:129], v120, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #5 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw4); (15,0,1,0:vw4); (15,0,2,0:vw4); (15,0,3,0:vw4) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v51, BufferOOB +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v51, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[16:17], v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v51, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b128 v[20:23], v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dwordx4 v[24:27], v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v51, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v18, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v51, v18, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[34:35], v18, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v19, v0, s60 +v_lshlrev_b32 v19, 0x2, v19 // Bias address scaled by BPE +v_cndmask_b32 v19, v51, v19, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v32, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v18, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v18, v51, v18, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v33, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v51, v33, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[42:43], v33, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v40, v0, s60 +v_lshlrev_b32 v40, 0x2, v40 // Bias address scaled by BPE +v_cndmask_b32 v40, v51, v40, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v41, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v33, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v33, v51, v33, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v48, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v51, v48, s[64:65] // LDC clip if OOB. offset +buffer_load_dwordx2 v[52:53], v48, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v49, v0, s60 +v_lshlrev_b32 v49, 0x2, v49 // Bias address scaled by BPE +v_cndmask_b32 v49, v51, v49, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v50, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v48, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v48, v51, v48, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+28], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+29], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+30], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+31], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+36], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+37], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+38], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+39], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+44], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+45], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+46], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+47], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+56], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+57], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+58], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+59], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 1, 0), (15, 0, 2, 0), (15, 0, 3, 0)] */ +v_mul_f32 v[vgprValuC+28], s[sgprAlpha], v[vgprValuC+28] // *= alpha +v_mul_f32 v[vgprValuC+29], s[sgprAlpha], v[vgprValuC+29] // *= alpha +v_mul_f32 v[vgprValuC+30], s[sgprAlpha], v[vgprValuC+30] // *= alpha +v_mul_f32 v[vgprValuC+31], s[sgprAlpha], v[vgprValuC+31] // *= alpha +v_mul_f32 v[vgprValuC+36], s[sgprAlpha], v[vgprValuC+36] // *= alpha +v_mul_f32 v[vgprValuC+37], s[sgprAlpha], v[vgprValuC+37] // *= alpha +v_mul_f32 v[vgprValuC+38], s[sgprAlpha], v[vgprValuC+38] // *= alpha +v_mul_f32 v[vgprValuC+39], s[sgprAlpha], v[vgprValuC+39] // *= alpha +v_mul_f32 v[vgprValuC+44], s[sgprAlpha], v[vgprValuC+44] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+46], s[sgprAlpha], v[vgprValuC+46] // *= alpha +v_mul_f32 v[vgprValuC+47], s[sgprAlpha], v[vgprValuC+47] // *= alpha +v_mul_f32 v[vgprValuC+56], s[sgprAlpha], v[vgprValuC+56] // *= alpha +v_mul_f32 v[vgprValuC+57], s[sgprAlpha], v[vgprValuC+57] // *= alpha +v_mul_f32 v[vgprValuC+58], s[sgprAlpha], v[vgprValuC+58] // *= alpha +v_mul_f32 v[vgprValuC+59], s[sgprAlpha], v[vgprValuC+59] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+28:vgprValuC+28+1], v[24:25], v[vgprValuC+28:vgprValuC+28+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+30:vgprValuC+30+1], v[26:27], v[vgprValuC+30:vgprValuC+30+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+28], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v16, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+29], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v17 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+30], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v17, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+31], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+28:vgprValuC+28+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+30:vgprValuC+30+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v28, v4 +v_mov_b32 v29, v5 +v_mov_b32 v30, v6 +v_mov_b32 v31, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+28], v[vgprValuC+28] // check Nan +v_bfe_u32 v9, v[vgprValuC+28], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+28], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+28], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+28], 16, v[vgprValuC+28] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+29], v[vgprValuC+29] // check Nan +v_bfe_u32 v9, v[vgprValuC+29], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+29], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+29], v9, v11, s[60:61] +v_and_or_b32 v28, v[vgprValuC+29], v10, v[vgprValuC+28] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+30], v[vgprValuC+30] // check Nan +v_bfe_u32 v9, v[vgprValuC+30], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+30], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+30], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+30], 16, v[vgprValuC+30] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+31], v[vgprValuC+31] // check Nan +v_bfe_u32 v9, v[vgprValuC+31], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+31], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+31], v9, v11, s[60:61] +v_and_or_b32 v29, v[vgprValuC+31], v10, v[vgprValuC+30] // pack two bf16 to dword +buffer_store_dwordx2 v[28:29], v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+36:vgprValuC+36+1], v[24:25], v[vgprValuC+36:vgprValuC+36+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+38:vgprValuC+38+1], v[26:27], v[vgprValuC+38:vgprValuC+38+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v34 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+36], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v34, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+37], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v35 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+38], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v35, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+39], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+36:vgprValuC+36+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+38:vgprValuC+38+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v36, v4 +v_mov_b32 v37, v5 +v_mov_b32 v38, v6 +v_mov_b32 v39, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+36], v[vgprValuC+36] // check Nan +v_bfe_u32 v9, v[vgprValuC+36], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+36], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+36], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+36], 16, v[vgprValuC+36] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+37], v[vgprValuC+37] // check Nan +v_bfe_u32 v9, v[vgprValuC+37], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+37], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+37], v9, v11, s[60:61] +v_and_or_b32 v36, v[vgprValuC+37], v10, v[vgprValuC+36] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+38], v[vgprValuC+38] // check Nan +v_bfe_u32 v9, v[vgprValuC+38], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+38], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+38], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+38], 16, v[vgprValuC+38] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+39], v[vgprValuC+39] // check Nan +v_bfe_u32 v9, v[vgprValuC+39], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+39], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+39], v9, v11, s[60:61] +v_and_or_b32 v37, v[vgprValuC+39], v10, v[vgprValuC+38] // pack two bf16 to dword +buffer_store_dwordx2 v[36:37], v18, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+44:vgprValuC+44+1], v[24:25], v[vgprValuC+44:vgprValuC+44+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+46:vgprValuC+46+1], v[26:27], v[vgprValuC+46:vgprValuC+46+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v42 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+44], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v42, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v43 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+46], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v43, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+47], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+44:vgprValuC+44+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+46:vgprValuC+46+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v44, v4 +v_mov_b32 v45, v5 +v_mov_b32 v46, v6 +v_mov_b32 v47, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+44], v[vgprValuC+44] // check Nan +v_bfe_u32 v9, v[vgprValuC+44], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+44], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+44], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+44], 16, v[vgprValuC+44] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_and_or_b32 v44, v[vgprValuC+45], v10, v[vgprValuC+44] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+46], v[vgprValuC+46] // check Nan +v_bfe_u32 v9, v[vgprValuC+46], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+46], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+46], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+46], 16, v[vgprValuC+46] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+47], v[vgprValuC+47] // check Nan +v_bfe_u32 v9, v[vgprValuC+47], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+47], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+47], v9, v11, s[60:61] +v_and_or_b32 v45, v[vgprValuC+47], v10, v[vgprValuC+46] // pack two bf16 to dword +buffer_store_dwordx2 v[44:45], v33, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v24, 1.0, v24, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+56:vgprValuC+56+1], v[24:25], v[vgprValuC+56:vgprValuC+56+1] // *= scaleAlphaVecVMulPK(24)(0) +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v26, 1.0, v26, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_cndmask_b32 v27, 1.0, v27, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_pk_mul_f32 v[vgprValuC+58:vgprValuC+58+1], v[26:27], v[vgprValuC+58:vgprValuC+58+1] // *= scaleAlphaVecVMulPK(24)(2) +v_lshlrev_b32 v4, 16, v52 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+56], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v52, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+57], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_lshlrev_b32 v4, 16, v53 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+58], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_and_b32 v4, v53, v10 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+59], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_pk_add_f32 v[4:5], v[20:21], v[vgprValuC+56:vgprValuC+56+1] // C += bias +v_pk_add_f32 v[6:7], v[22:23], v[vgprValuC+58:vgprValuC+58+1] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v56, v4 +v_mov_b32 v57, v5 +v_mov_b32 v58, v6 +v_mov_b32 v59, v7 +v_cmp_u_f32 s[60:61], v[vgprValuC+56], v[vgprValuC+56] // check Nan +v_bfe_u32 v9, v[vgprValuC+56], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+56], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+56], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+56], 16, v[vgprValuC+56] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+57], v[vgprValuC+57] // check Nan +v_bfe_u32 v9, v[vgprValuC+57], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+57], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+57], v9, v11, s[60:61] +v_and_or_b32 v56, v[vgprValuC+57], v10, v[vgprValuC+56] // pack two bf16 to dword +v_cmp_u_f32 s[60:61], v[vgprValuC+58], v[vgprValuC+58] // check Nan +v_bfe_u32 v9, v[vgprValuC+58], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+58], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+58], v9, v11, s[60:61] +v_lshrrev_b32 v[vgprValuC+58], 16, v[vgprValuC+58] // convert C to bf16 +v_cmp_u_f32 s[60:61], v[vgprValuC+59], v[vgprValuC+59] // check Nan +v_bfe_u32 v9, v[vgprValuC+59], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+59], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+59], v9, v11, s[60:61] +v_and_or_b32 v57, v[vgprValuC+59], v10, v[vgprValuC+58] // pack two bf16 to dword +buffer_store_dwordx2 v[56:57], v48, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_2 // jump to end +label_GW_B1_E1_M: +s_cmpk_eq_u32 s[sgprActivationType], 1 // activationType == 1 +s_cbranch_scc1 label_To_Activation_Abs_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 2 // activationType == 2 +s_cbranch_scc1 label_To_Activation_Clippedrelu_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 3 // activationType == 3 +s_cbranch_scc1 label_To_Activation_Gelu_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 4 // activationType == 4 +s_cbranch_scc1 label_To_Activation_Leakyrelu_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 5 // activationType == 5 +s_cbranch_scc1 label_To_Activation_Relu_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 6 // activationType == 6 +s_cbranch_scc1 label_To_Activation_Sigmoid_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 7 // activationType == 7 +s_cbranch_scc1 label_To_Activation_Tanh_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 9 // activationType == 9 +s_cbranch_scc1 label_To_Activation_Geluscaling_VW1_beta_1_edge_1 // Branch if true +s_cmpk_eq_u32 s[sgprActivationType], 10 // activationType == 10 +s_cbranch_scc1 label_To_Activation_Silu_VW1_beta_1_edge_1 // Branch if true +label_To_Activation_None_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_None_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Abs_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Abs_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Clippedrelu_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Clippedrelu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Gelu_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Gelu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Leakyrelu_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Leakyrelu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Relu_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Relu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Sigmoid_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Sigmoid_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Tanh_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Tanh_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Geluscaling_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Geluscaling_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_To_Activation_Silu_VW1_beta_1_edge_1: +s_getpc_b64 s[12:13] // addr of next instr +s_add_i32 s8, label_Activation_Silu_VW1, 0x4 // target branch offset +s_add_u32 s12, s12, s8 // add target branch offset +s_addc_u32 s13, s13, 0 // add high and carry +s_branch label_ActivationSetPCAddrEnd_1 +label_ActivationSetPCAddrEnd_1: + +/* edge=1, allocate 6 sgpr. perBatchTmpS=4 perBatchMaskS=2 perElementMaskS=0 elementsPerBatch=16 */ +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #0 (d1,d0,vc1,vc0) = */ +/* (0,0,0,0:vw1); (0,0,0,1:vw1); (0,0,0,2:vw1); (0,0,0,3:vw1); (0,0,1,0:vw1); (0,0,1,1:vw1); (0,0,1,2:vw1); (0,0,1,3:vw1); (0,0,2,0:vw1); (0,0,2,1:vw1); (0,0,2,2:vw1); (0,0,2,3:vw1); (0,0,3,0:vw1); (0,0,3,1:vw1); (0,0,3,2:vw1); (0,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(0,0,0,0) */ +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +s_waitcnt lgkmcnt(0) // Wait for Bias LDS write +s_barrier // Bias LDS write barrier +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(0,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc0 // copy acc to vreg[0] +v_accvgpr_read_b32 v[vgprValuC+26], acc4 // copy acc to vreg[1] +v_accvgpr_read_b32 v[vgprValuC+33], acc8 // copy acc to vreg[2] +v_accvgpr_read_b32 v[vgprValuC+40], acc12 // copy acc to vreg[3] +v_accvgpr_read_b32 v[vgprValuC+45], acc1 // copy acc to vreg[4] +v_accvgpr_read_b32 v[vgprValuC+50], acc5 // copy acc to vreg[5] +v_accvgpr_read_b32 v[vgprValuC+55], acc9 // copy acc to vreg[6] +v_accvgpr_read_b32 v[vgprValuC+60], acc13 // copy acc to vreg[7] +v_accvgpr_read_b32 v[vgprValuC+65], acc2 // copy acc to vreg[8] +v_accvgpr_read_b32 v[vgprValuC+70], acc6 // copy acc to vreg[9] +v_accvgpr_read_b32 v[vgprValuC+75], acc10 // copy acc to vreg[10] +v_accvgpr_read_b32 v[vgprValuC+80], acc14 // copy acc to vreg[11] +v_accvgpr_read_b32 v[vgprValuC+85], acc3 // copy acc to vreg[12] +v_accvgpr_read_b32 v[vgprValuC+90], acc7 // copy acc to vreg[13] +v_accvgpr_read_b32 v[vgprValuC+95], acc11 // copy acc to vreg[14] +v_accvgpr_read_b32 v[vgprValuC+100], acc15 // copy acc to vreg[15] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(0, 0, 0, 0), (0, 0, 0, 1), (0, 0, 0, 2), (0, 0, 0, 3), (0, 0, 1, 0), (0, 0, 1, 1), (0, 0, 1, 2), (0, 0, 1, 3), (0, 0, 2, 0), (0, 0, 2, 1), (0, 0, 2, 2), (0, 0, 2, 3), (0, 0, 3, 0), (0, 0, 3, 1), (0, 0, 3, 2), (0, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #1 (d1,d0,vc1,vc0) = */ +/* (1,0,0,0:vw1); (1,0,0,1:vw1); (1,0,0,2:vw1); (1,0,0,3:vw1); (1,0,1,0:vw1); (1,0,1,1:vw1); (1,0,1,2:vw1); (1,0,1,3:vw1); (1,0,2,0:vw1); (1,0,2,1:vw1); (1,0,2,2:vw1); (1,0,2,3:vw1); (1,0,3,0:vw1); (1,0,3,1:vw1); (1,0,3,2:vw1); (1,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(1,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(1,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc16 // copy acc to vreg[16] +v_accvgpr_read_b32 v[vgprValuC+26], acc20 // copy acc to vreg[17] +v_accvgpr_read_b32 v[vgprValuC+33], acc24 // copy acc to vreg[18] +v_accvgpr_read_b32 v[vgprValuC+40], acc28 // copy acc to vreg[19] +v_accvgpr_read_b32 v[vgprValuC+45], acc17 // copy acc to vreg[20] +v_accvgpr_read_b32 v[vgprValuC+50], acc21 // copy acc to vreg[21] +v_accvgpr_read_b32 v[vgprValuC+55], acc25 // copy acc to vreg[22] +v_accvgpr_read_b32 v[vgprValuC+60], acc29 // copy acc to vreg[23] +v_accvgpr_read_b32 v[vgprValuC+65], acc18 // copy acc to vreg[24] +v_accvgpr_read_b32 v[vgprValuC+70], acc22 // copy acc to vreg[25] +v_accvgpr_read_b32 v[vgprValuC+75], acc26 // copy acc to vreg[26] +v_accvgpr_read_b32 v[vgprValuC+80], acc30 // copy acc to vreg[27] +v_accvgpr_read_b32 v[vgprValuC+85], acc19 // copy acc to vreg[28] +v_accvgpr_read_b32 v[vgprValuC+90], acc23 // copy acc to vreg[29] +v_accvgpr_read_b32 v[vgprValuC+95], acc27 // copy acc to vreg[30] +v_accvgpr_read_b32 v[vgprValuC+100], acc31 // copy acc to vreg[31] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(1, 0, 0, 0), (1, 0, 0, 1), (1, 0, 0, 2), (1, 0, 0, 3), (1, 0, 1, 0), (1, 0, 1, 1), (1, 0, 1, 2), (1, 0, 1, 3), (1, 0, 2, 0), (1, 0, 2, 1), (1, 0, 2, 2), (1, 0, 2, 3), (1, 0, 3, 0), (1, 0, 3, 1), (1, 0, 3, 2), (1, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #2 (d1,d0,vc1,vc0) = */ +/* (2,0,0,0:vw1); (2,0,0,1:vw1); (2,0,0,2:vw1); (2,0,0,3:vw1); (2,0,1,0:vw1); (2,0,1,1:vw1); (2,0,1,2:vw1); (2,0,1,3:vw1); (2,0,2,0:vw1); (2,0,2,1:vw1); (2,0,2,2:vw1); (2,0,2,3:vw1); (2,0,3,0:vw1); (2,0,3,1:vw1); (2,0,3,2:vw1); (2,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(2,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(2,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc32 // copy acc to vreg[32] +v_accvgpr_read_b32 v[vgprValuC+26], acc36 // copy acc to vreg[33] +v_accvgpr_read_b32 v[vgprValuC+33], acc40 // copy acc to vreg[34] +v_accvgpr_read_b32 v[vgprValuC+40], acc44 // copy acc to vreg[35] +v_accvgpr_read_b32 v[vgprValuC+45], acc33 // copy acc to vreg[36] +v_accvgpr_read_b32 v[vgprValuC+50], acc37 // copy acc to vreg[37] +v_accvgpr_read_b32 v[vgprValuC+55], acc41 // copy acc to vreg[38] +v_accvgpr_read_b32 v[vgprValuC+60], acc45 // copy acc to vreg[39] +v_accvgpr_read_b32 v[vgprValuC+65], acc34 // copy acc to vreg[40] +v_accvgpr_read_b32 v[vgprValuC+70], acc38 // copy acc to vreg[41] +v_accvgpr_read_b32 v[vgprValuC+75], acc42 // copy acc to vreg[42] +v_accvgpr_read_b32 v[vgprValuC+80], acc46 // copy acc to vreg[43] +v_accvgpr_read_b32 v[vgprValuC+85], acc35 // copy acc to vreg[44] +v_accvgpr_read_b32 v[vgprValuC+90], acc39 // copy acc to vreg[45] +v_accvgpr_read_b32 v[vgprValuC+95], acc43 // copy acc to vreg[46] +v_accvgpr_read_b32 v[vgprValuC+100], acc47 // copy acc to vreg[47] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(2, 0, 0, 0), (2, 0, 0, 1), (2, 0, 0, 2), (2, 0, 0, 3), (2, 0, 1, 0), (2, 0, 1, 1), (2, 0, 1, 2), (2, 0, 1, 3), (2, 0, 2, 0), (2, 0, 2, 1), (2, 0, 2, 2), (2, 0, 2, 3), (2, 0, 3, 0), (2, 0, 3, 1), (2, 0, 3, 2), (2, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #3 (d1,d0,vc1,vc0) = */ +/* (3,0,0,0:vw1); (3,0,0,1:vw1); (3,0,0,2:vw1); (3,0,0,3:vw1); (3,0,1,0:vw1); (3,0,1,1:vw1); (3,0,1,2:vw1); (3,0,1,3:vw1); (3,0,2,0:vw1); (3,0,2,1:vw1); (3,0,2,2:vw1); (3,0,2,3:vw1); (3,0,3,0:vw1); (3,0,3,1:vw1); (3,0,3,2:vw1); (3,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(3,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(3,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc48 // copy acc to vreg[48] +v_accvgpr_read_b32 v[vgprValuC+26], acc52 // copy acc to vreg[49] +v_accvgpr_read_b32 v[vgprValuC+33], acc56 // copy acc to vreg[50] +v_accvgpr_read_b32 v[vgprValuC+40], acc60 // copy acc to vreg[51] +v_accvgpr_read_b32 v[vgprValuC+45], acc49 // copy acc to vreg[52] +v_accvgpr_read_b32 v[vgprValuC+50], acc53 // copy acc to vreg[53] +v_accvgpr_read_b32 v[vgprValuC+55], acc57 // copy acc to vreg[54] +v_accvgpr_read_b32 v[vgprValuC+60], acc61 // copy acc to vreg[55] +v_accvgpr_read_b32 v[vgprValuC+65], acc50 // copy acc to vreg[56] +v_accvgpr_read_b32 v[vgprValuC+70], acc54 // copy acc to vreg[57] +v_accvgpr_read_b32 v[vgprValuC+75], acc58 // copy acc to vreg[58] +v_accvgpr_read_b32 v[vgprValuC+80], acc62 // copy acc to vreg[59] +v_accvgpr_read_b32 v[vgprValuC+85], acc51 // copy acc to vreg[60] +v_accvgpr_read_b32 v[vgprValuC+90], acc55 // copy acc to vreg[61] +v_accvgpr_read_b32 v[vgprValuC+95], acc59 // copy acc to vreg[62] +v_accvgpr_read_b32 v[vgprValuC+100], acc63 // copy acc to vreg[63] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(3, 0, 0, 0), (3, 0, 0, 1), (3, 0, 0, 2), (3, 0, 0, 3), (3, 0, 1, 0), (3, 0, 1, 1), (3, 0, 1, 2), (3, 0, 1, 3), (3, 0, 2, 0), (3, 0, 2, 1), (3, 0, 2, 2), (3, 0, 2, 3), (3, 0, 3, 0), (3, 0, 3, 1), (3, 0, 3, 2), (3, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #4 (d1,d0,vc1,vc0) = */ +/* (4,0,0,0:vw1); (4,0,0,1:vw1); (4,0,0,2:vw1); (4,0,0,3:vw1); (4,0,1,0:vw1); (4,0,1,1:vw1); (4,0,1,2:vw1); (4,0,1,3:vw1); (4,0,2,0:vw1); (4,0,2,1:vw1); (4,0,2,2:vw1); (4,0,2,3:vw1); (4,0,3,0:vw1); (4,0,3,1:vw1); (4,0,3,2:vw1); (4,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(4,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(4,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc64 // copy acc to vreg[64] +v_accvgpr_read_b32 v[vgprValuC+26], acc68 // copy acc to vreg[65] +v_accvgpr_read_b32 v[vgprValuC+33], acc72 // copy acc to vreg[66] +v_accvgpr_read_b32 v[vgprValuC+40], acc76 // copy acc to vreg[67] +v_accvgpr_read_b32 v[vgprValuC+45], acc65 // copy acc to vreg[68] +v_accvgpr_read_b32 v[vgprValuC+50], acc69 // copy acc to vreg[69] +v_accvgpr_read_b32 v[vgprValuC+55], acc73 // copy acc to vreg[70] +v_accvgpr_read_b32 v[vgprValuC+60], acc77 // copy acc to vreg[71] +v_accvgpr_read_b32 v[vgprValuC+65], acc66 // copy acc to vreg[72] +v_accvgpr_read_b32 v[vgprValuC+70], acc70 // copy acc to vreg[73] +v_accvgpr_read_b32 v[vgprValuC+75], acc74 // copy acc to vreg[74] +v_accvgpr_read_b32 v[vgprValuC+80], acc78 // copy acc to vreg[75] +v_accvgpr_read_b32 v[vgprValuC+85], acc67 // copy acc to vreg[76] +v_accvgpr_read_b32 v[vgprValuC+90], acc71 // copy acc to vreg[77] +v_accvgpr_read_b32 v[vgprValuC+95], acc75 // copy acc to vreg[78] +v_accvgpr_read_b32 v[vgprValuC+100], acc79 // copy acc to vreg[79] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(4, 0, 0, 0), (4, 0, 0, 1), (4, 0, 0, 2), (4, 0, 0, 3), (4, 0, 1, 0), (4, 0, 1, 1), (4, 0, 1, 2), (4, 0, 1, 3), (4, 0, 2, 0), (4, 0, 2, 1), (4, 0, 2, 2), (4, 0, 2, 3), (4, 0, 3, 0), (4, 0, 3, 1), (4, 0, 3, 2), (4, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #5 (d1,d0,vc1,vc0) = */ +/* (5,0,0,0:vw1); (5,0,0,1:vw1); (5,0,0,2:vw1); (5,0,0,3:vw1); (5,0,1,0:vw1); (5,0,1,1:vw1); (5,0,1,2:vw1); (5,0,1,3:vw1); (5,0,2,0:vw1); (5,0,2,1:vw1); (5,0,2,2:vw1); (5,0,2,3:vw1); (5,0,3,0:vw1); (5,0,3,1:vw1); (5,0,3,2:vw1); (5,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(5,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(5,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc80 // copy acc to vreg[80] +v_accvgpr_read_b32 v[vgprValuC+26], acc84 // copy acc to vreg[81] +v_accvgpr_read_b32 v[vgprValuC+33], acc88 // copy acc to vreg[82] +v_accvgpr_read_b32 v[vgprValuC+40], acc92 // copy acc to vreg[83] +v_accvgpr_read_b32 v[vgprValuC+45], acc81 // copy acc to vreg[84] +v_accvgpr_read_b32 v[vgprValuC+50], acc85 // copy acc to vreg[85] +v_accvgpr_read_b32 v[vgprValuC+55], acc89 // copy acc to vreg[86] +v_accvgpr_read_b32 v[vgprValuC+60], acc93 // copy acc to vreg[87] +v_accvgpr_read_b32 v[vgprValuC+65], acc82 // copy acc to vreg[88] +v_accvgpr_read_b32 v[vgprValuC+70], acc86 // copy acc to vreg[89] +v_accvgpr_read_b32 v[vgprValuC+75], acc90 // copy acc to vreg[90] +v_accvgpr_read_b32 v[vgprValuC+80], acc94 // copy acc to vreg[91] +v_accvgpr_read_b32 v[vgprValuC+85], acc83 // copy acc to vreg[92] +v_accvgpr_read_b32 v[vgprValuC+90], acc87 // copy acc to vreg[93] +v_accvgpr_read_b32 v[vgprValuC+95], acc91 // copy acc to vreg[94] +v_accvgpr_read_b32 v[vgprValuC+100], acc95 // copy acc to vreg[95] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(5, 0, 0, 0), (5, 0, 0, 1), (5, 0, 0, 2), (5, 0, 0, 3), (5, 0, 1, 0), (5, 0, 1, 1), (5, 0, 1, 2), (5, 0, 1, 3), (5, 0, 2, 0), (5, 0, 2, 1), (5, 0, 2, 2), (5, 0, 2, 3), (5, 0, 3, 0), (5, 0, 3, 1), (5, 0, 3, 2), (5, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #6 (d1,d0,vc1,vc0) = */ +/* (6,0,0,0:vw1); (6,0,0,1:vw1); (6,0,0,2:vw1); (6,0,0,3:vw1); (6,0,1,0:vw1); (6,0,1,1:vw1); (6,0,1,2:vw1); (6,0,1,3:vw1); (6,0,2,0:vw1); (6,0,2,1:vw1); (6,0,2,2:vw1); (6,0,2,3:vw1); (6,0,3,0:vw1); (6,0,3,1:vw1); (6,0,3,2:vw1); (6,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(6,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(6,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc96 // copy acc to vreg[96] +v_accvgpr_read_b32 v[vgprValuC+26], acc100 // copy acc to vreg[97] +v_accvgpr_read_b32 v[vgprValuC+33], acc104 // copy acc to vreg[98] +v_accvgpr_read_b32 v[vgprValuC+40], acc108 // copy acc to vreg[99] +v_accvgpr_read_b32 v[vgprValuC+45], acc97 // copy acc to vreg[100] +v_accvgpr_read_b32 v[vgprValuC+50], acc101 // copy acc to vreg[101] +v_accvgpr_read_b32 v[vgprValuC+55], acc105 // copy acc to vreg[102] +v_accvgpr_read_b32 v[vgprValuC+60], acc109 // copy acc to vreg[103] +v_accvgpr_read_b32 v[vgprValuC+65], acc98 // copy acc to vreg[104] +v_accvgpr_read_b32 v[vgprValuC+70], acc102 // copy acc to vreg[105] +v_accvgpr_read_b32 v[vgprValuC+75], acc106 // copy acc to vreg[106] +v_accvgpr_read_b32 v[vgprValuC+80], acc110 // copy acc to vreg[107] +v_accvgpr_read_b32 v[vgprValuC+85], acc99 // copy acc to vreg[108] +v_accvgpr_read_b32 v[vgprValuC+90], acc103 // copy acc to vreg[109] +v_accvgpr_read_b32 v[vgprValuC+95], acc107 // copy acc to vreg[110] +v_accvgpr_read_b32 v[vgprValuC+100], acc111 // copy acc to vreg[111] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(6, 0, 0, 0), (6, 0, 0, 1), (6, 0, 0, 2), (6, 0, 0, 3), (6, 0, 1, 0), (6, 0, 1, 1), (6, 0, 1, 2), (6, 0, 1, 3), (6, 0, 2, 0), (6, 0, 2, 1), (6, 0, 2, 2), (6, 0, 2, 3), (6, 0, 3, 0), (6, 0, 3, 1), (6, 0, 3, 2), (6, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #7 (d1,d0,vc1,vc0) = */ +/* (7,0,0,0:vw1); (7,0,0,1:vw1); (7,0,0,2:vw1); (7,0,0,3:vw1); (7,0,1,0:vw1); (7,0,1,1:vw1); (7,0,1,2:vw1); (7,0,1,3:vw1); (7,0,2,0:vw1); (7,0,2,1:vw1); (7,0,2,2:vw1); (7,0,2,3:vw1); (7,0,3,0:vw1); (7,0,3,1:vw1); (7,0,3,2:vw1); (7,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(7,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(7,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc112 // copy acc to vreg[112] +v_accvgpr_read_b32 v[vgprValuC+26], acc116 // copy acc to vreg[113] +v_accvgpr_read_b32 v[vgprValuC+33], acc120 // copy acc to vreg[114] +v_accvgpr_read_b32 v[vgprValuC+40], acc124 // copy acc to vreg[115] +v_accvgpr_read_b32 v[vgprValuC+45], acc113 // copy acc to vreg[116] +v_accvgpr_read_b32 v[vgprValuC+50], acc117 // copy acc to vreg[117] +v_accvgpr_read_b32 v[vgprValuC+55], acc121 // copy acc to vreg[118] +v_accvgpr_read_b32 v[vgprValuC+60], acc125 // copy acc to vreg[119] +v_accvgpr_read_b32 v[vgprValuC+65], acc114 // copy acc to vreg[120] +v_accvgpr_read_b32 v[vgprValuC+70], acc118 // copy acc to vreg[121] +v_accvgpr_read_b32 v[vgprValuC+75], acc122 // copy acc to vreg[122] +v_accvgpr_read_b32 v[vgprValuC+80], acc126 // copy acc to vreg[123] +v_accvgpr_read_b32 v[vgprValuC+85], acc115 // copy acc to vreg[124] +v_accvgpr_read_b32 v[vgprValuC+90], acc119 // copy acc to vreg[125] +v_accvgpr_read_b32 v[vgprValuC+95], acc123 // copy acc to vreg[126] +v_accvgpr_read_b32 v[vgprValuC+100], acc127 // copy acc to vreg[127] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(7, 0, 0, 0), (7, 0, 0, 1), (7, 0, 0, 2), (7, 0, 0, 3), (7, 0, 1, 0), (7, 0, 1, 1), (7, 0, 1, 2), (7, 0, 1, 3), (7, 0, 2, 0), (7, 0, 2, 1), (7, 0, 2, 2), (7, 0, 2, 3), (7, 0, 3, 0), (7, 0, 3, 1), (7, 0, 3, 2), (7, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #8 (d1,d0,vc1,vc0) = */ +/* (8,0,0,0:vw1); (8,0,0,1:vw1); (8,0,0,2:vw1); (8,0,0,3:vw1); (8,0,1,0:vw1); (8,0,1,1:vw1); (8,0,1,2:vw1); (8,0,1,3:vw1); (8,0,2,0:vw1); (8,0,2,1:vw1); (8,0,2,2:vw1); (8,0,2,3:vw1); (8,0,3,0:vw1); (8,0,3,1:vw1); (8,0,3,2:vw1); (8,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(8,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(8,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc128 // copy acc to vreg[128] +v_accvgpr_read_b32 v[vgprValuC+26], acc132 // copy acc to vreg[129] +v_accvgpr_read_b32 v[vgprValuC+33], acc136 // copy acc to vreg[130] +v_accvgpr_read_b32 v[vgprValuC+40], acc140 // copy acc to vreg[131] +v_accvgpr_read_b32 v[vgprValuC+45], acc129 // copy acc to vreg[132] +v_accvgpr_read_b32 v[vgprValuC+50], acc133 // copy acc to vreg[133] +v_accvgpr_read_b32 v[vgprValuC+55], acc137 // copy acc to vreg[134] +v_accvgpr_read_b32 v[vgprValuC+60], acc141 // copy acc to vreg[135] +v_accvgpr_read_b32 v[vgprValuC+65], acc130 // copy acc to vreg[136] +v_accvgpr_read_b32 v[vgprValuC+70], acc134 // copy acc to vreg[137] +v_accvgpr_read_b32 v[vgprValuC+75], acc138 // copy acc to vreg[138] +v_accvgpr_read_b32 v[vgprValuC+80], acc142 // copy acc to vreg[139] +v_accvgpr_read_b32 v[vgprValuC+85], acc131 // copy acc to vreg[140] +v_accvgpr_read_b32 v[vgprValuC+90], acc135 // copy acc to vreg[141] +v_accvgpr_read_b32 v[vgprValuC+95], acc139 // copy acc to vreg[142] +v_accvgpr_read_b32 v[vgprValuC+100], acc143 // copy acc to vreg[143] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(8, 0, 0, 0), (8, 0, 0, 1), (8, 0, 0, 2), (8, 0, 0, 3), (8, 0, 1, 0), (8, 0, 1, 1), (8, 0, 1, 2), (8, 0, 1, 3), (8, 0, 2, 0), (8, 0, 2, 1), (8, 0, 2, 2), (8, 0, 2, 3), (8, 0, 3, 0), (8, 0, 3, 1), (8, 0, 3, 2), (8, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #9 (d1,d0,vc1,vc0) = */ +/* (9,0,0,0:vw1); (9,0,0,1:vw1); (9,0,0,2:vw1); (9,0,0,3:vw1); (9,0,1,0:vw1); (9,0,1,1:vw1); (9,0,1,2:vw1); (9,0,1,3:vw1); (9,0,2,0:vw1); (9,0,2,1:vw1); (9,0,2,2:vw1); (9,0,2,3:vw1); (9,0,3,0:vw1); (9,0,3,1:vw1); (9,0,3,2:vw1); (9,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(9,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(9,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc144 // copy acc to vreg[144] +v_accvgpr_read_b32 v[vgprValuC+26], acc148 // copy acc to vreg[145] +v_accvgpr_read_b32 v[vgprValuC+33], acc152 // copy acc to vreg[146] +v_accvgpr_read_b32 v[vgprValuC+40], acc156 // copy acc to vreg[147] +v_accvgpr_read_b32 v[vgprValuC+45], acc145 // copy acc to vreg[148] +v_accvgpr_read_b32 v[vgprValuC+50], acc149 // copy acc to vreg[149] +v_accvgpr_read_b32 v[vgprValuC+55], acc153 // copy acc to vreg[150] +v_accvgpr_read_b32 v[vgprValuC+60], acc157 // copy acc to vreg[151] +v_accvgpr_read_b32 v[vgprValuC+65], acc146 // copy acc to vreg[152] +v_accvgpr_read_b32 v[vgprValuC+70], acc150 // copy acc to vreg[153] +v_accvgpr_read_b32 v[vgprValuC+75], acc154 // copy acc to vreg[154] +v_accvgpr_read_b32 v[vgprValuC+80], acc158 // copy acc to vreg[155] +v_accvgpr_read_b32 v[vgprValuC+85], acc147 // copy acc to vreg[156] +v_accvgpr_read_b32 v[vgprValuC+90], acc151 // copy acc to vreg[157] +v_accvgpr_read_b32 v[vgprValuC+95], acc155 // copy acc to vreg[158] +v_accvgpr_read_b32 v[vgprValuC+100], acc159 // copy acc to vreg[159] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(9, 0, 0, 0), (9, 0, 0, 1), (9, 0, 0, 2), (9, 0, 0, 3), (9, 0, 1, 0), (9, 0, 1, 1), (9, 0, 1, 2), (9, 0, 1, 3), (9, 0, 2, 0), (9, 0, 2, 1), (9, 0, 2, 2), (9, 0, 2, 3), (9, 0, 3, 0), (9, 0, 3, 1), (9, 0, 3, 2), (9, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #10 (d1,d0,vc1,vc0) = */ +/* (10,0,0,0:vw1); (10,0,0,1:vw1); (10,0,0,2:vw1); (10,0,0,3:vw1); (10,0,1,0:vw1); (10,0,1,1:vw1); (10,0,1,2:vw1); (10,0,1,3:vw1); (10,0,2,0:vw1); (10,0,2,1:vw1); (10,0,2,2:vw1); (10,0,2,3:vw1); (10,0,3,0:vw1); (10,0,3,1:vw1); (10,0,3,2:vw1); (10,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(10,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(10,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc160 // copy acc to vreg[160] +v_accvgpr_read_b32 v[vgprValuC+26], acc164 // copy acc to vreg[161] +v_accvgpr_read_b32 v[vgprValuC+33], acc168 // copy acc to vreg[162] +v_accvgpr_read_b32 v[vgprValuC+40], acc172 // copy acc to vreg[163] +v_accvgpr_read_b32 v[vgprValuC+45], acc161 // copy acc to vreg[164] +v_accvgpr_read_b32 v[vgprValuC+50], acc165 // copy acc to vreg[165] +v_accvgpr_read_b32 v[vgprValuC+55], acc169 // copy acc to vreg[166] +v_accvgpr_read_b32 v[vgprValuC+60], acc173 // copy acc to vreg[167] +v_accvgpr_read_b32 v[vgprValuC+65], acc162 // copy acc to vreg[168] +v_accvgpr_read_b32 v[vgprValuC+70], acc166 // copy acc to vreg[169] +v_accvgpr_read_b32 v[vgprValuC+75], acc170 // copy acc to vreg[170] +v_accvgpr_read_b32 v[vgprValuC+80], acc174 // copy acc to vreg[171] +v_accvgpr_read_b32 v[vgprValuC+85], acc163 // copy acc to vreg[172] +v_accvgpr_read_b32 v[vgprValuC+90], acc167 // copy acc to vreg[173] +v_accvgpr_read_b32 v[vgprValuC+95], acc171 // copy acc to vreg[174] +v_accvgpr_read_b32 v[vgprValuC+100], acc175 // copy acc to vreg[175] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(10, 0, 0, 0), (10, 0, 0, 1), (10, 0, 0, 2), (10, 0, 0, 3), (10, 0, 1, 0), (10, 0, 1, 1), (10, 0, 1, 2), (10, 0, 1, 3), (10, 0, 2, 0), (10, 0, 2, 1), (10, 0, 2, 2), (10, 0, 2, 3), (10, 0, 3, 0), (10, 0, 3, 1), (10, 0, 3, 2), (10, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #11 (d1,d0,vc1,vc0) = */ +/* (11,0,0,0:vw1); (11,0,0,1:vw1); (11,0,0,2:vw1); (11,0,0,3:vw1); (11,0,1,0:vw1); (11,0,1,1:vw1); (11,0,1,2:vw1); (11,0,1,3:vw1); (11,0,2,0:vw1); (11,0,2,1:vw1); (11,0,2,2:vw1); (11,0,2,3:vw1); (11,0,3,0:vw1); (11,0,3,1:vw1); (11,0,3,2:vw1); (11,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(11,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(11,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc176 // copy acc to vreg[176] +v_accvgpr_read_b32 v[vgprValuC+26], acc180 // copy acc to vreg[177] +v_accvgpr_read_b32 v[vgprValuC+33], acc184 // copy acc to vreg[178] +v_accvgpr_read_b32 v[vgprValuC+40], acc188 // copy acc to vreg[179] +v_accvgpr_read_b32 v[vgprValuC+45], acc177 // copy acc to vreg[180] +v_accvgpr_read_b32 v[vgprValuC+50], acc181 // copy acc to vreg[181] +v_accvgpr_read_b32 v[vgprValuC+55], acc185 // copy acc to vreg[182] +v_accvgpr_read_b32 v[vgprValuC+60], acc189 // copy acc to vreg[183] +v_accvgpr_read_b32 v[vgprValuC+65], acc178 // copy acc to vreg[184] +v_accvgpr_read_b32 v[vgprValuC+70], acc182 // copy acc to vreg[185] +v_accvgpr_read_b32 v[vgprValuC+75], acc186 // copy acc to vreg[186] +v_accvgpr_read_b32 v[vgprValuC+80], acc190 // copy acc to vreg[187] +v_accvgpr_read_b32 v[vgprValuC+85], acc179 // copy acc to vreg[188] +v_accvgpr_read_b32 v[vgprValuC+90], acc183 // copy acc to vreg[189] +v_accvgpr_read_b32 v[vgprValuC+95], acc187 // copy acc to vreg[190] +v_accvgpr_read_b32 v[vgprValuC+100], acc191 // copy acc to vreg[191] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(11, 0, 0, 0), (11, 0, 0, 1), (11, 0, 0, 2), (11, 0, 0, 3), (11, 0, 1, 0), (11, 0, 1, 1), (11, 0, 1, 2), (11, 0, 1, 3), (11, 0, 2, 0), (11, 0, 2, 1), (11, 0, 2, 2), (11, 0, 2, 3), (11, 0, 3, 0), (11, 0, 3, 1), (11, 0, 3, 2), (11, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #12 (d1,d0,vc1,vc0) = */ +/* (12,0,0,0:vw1); (12,0,0,1:vw1); (12,0,0,2:vw1); (12,0,0,3:vw1); (12,0,1,0:vw1); (12,0,1,1:vw1); (12,0,1,2:vw1); (12,0,1,3:vw1); (12,0,2,0:vw1); (12,0,2,1:vw1); (12,0,2,2:vw1); (12,0,2,3:vw1); (12,0,3,0:vw1); (12,0,3,1:vw1); (12,0,3,2:vw1); (12,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(12,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(12,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc192 // copy acc to vreg[192] +v_accvgpr_read_b32 v[vgprValuC+26], acc196 // copy acc to vreg[193] +v_accvgpr_read_b32 v[vgprValuC+33], acc200 // copy acc to vreg[194] +v_accvgpr_read_b32 v[vgprValuC+40], acc204 // copy acc to vreg[195] +v_accvgpr_read_b32 v[vgprValuC+45], acc193 // copy acc to vreg[196] +v_accvgpr_read_b32 v[vgprValuC+50], acc197 // copy acc to vreg[197] +v_accvgpr_read_b32 v[vgprValuC+55], acc201 // copy acc to vreg[198] +v_accvgpr_read_b32 v[vgprValuC+60], acc205 // copy acc to vreg[199] +v_accvgpr_read_b32 v[vgprValuC+65], acc194 // copy acc to vreg[200] +v_accvgpr_read_b32 v[vgprValuC+70], acc198 // copy acc to vreg[201] +v_accvgpr_read_b32 v[vgprValuC+75], acc202 // copy acc to vreg[202] +v_accvgpr_read_b32 v[vgprValuC+80], acc206 // copy acc to vreg[203] +v_accvgpr_read_b32 v[vgprValuC+85], acc195 // copy acc to vreg[204] +v_accvgpr_read_b32 v[vgprValuC+90], acc199 // copy acc to vreg[205] +v_accvgpr_read_b32 v[vgprValuC+95], acc203 // copy acc to vreg[206] +v_accvgpr_read_b32 v[vgprValuC+100], acc207 // copy acc to vreg[207] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(12, 0, 0, 0), (12, 0, 0, 1), (12, 0, 0, 2), (12, 0, 0, 3), (12, 0, 1, 0), (12, 0, 1, 1), (12, 0, 1, 2), (12, 0, 1, 3), (12, 0, 2, 0), (12, 0, 2, 1), (12, 0, 2, 2), (12, 0, 2, 3), (12, 0, 3, 0), (12, 0, 3, 1), (12, 0, 3, 2), (12, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #13 (d1,d0,vc1,vc0) = */ +/* (13,0,0,0:vw1); (13,0,0,1:vw1); (13,0,0,2:vw1); (13,0,0,3:vw1); (13,0,1,0:vw1); (13,0,1,1:vw1); (13,0,1,2:vw1); (13,0,1,3:vw1); (13,0,2,0:vw1); (13,0,2,1:vw1); (13,0,2,2:vw1); (13,0,2,3:vw1); (13,0,3,0:vw1); (13,0,3,1:vw1); (13,0,3,2:vw1); (13,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(13,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(13,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc208 // copy acc to vreg[208] +v_accvgpr_read_b32 v[vgprValuC+26], acc212 // copy acc to vreg[209] +v_accvgpr_read_b32 v[vgprValuC+33], acc216 // copy acc to vreg[210] +v_accvgpr_read_b32 v[vgprValuC+40], acc220 // copy acc to vreg[211] +v_accvgpr_read_b32 v[vgprValuC+45], acc209 // copy acc to vreg[212] +v_accvgpr_read_b32 v[vgprValuC+50], acc213 // copy acc to vreg[213] +v_accvgpr_read_b32 v[vgprValuC+55], acc217 // copy acc to vreg[214] +v_accvgpr_read_b32 v[vgprValuC+60], acc221 // copy acc to vreg[215] +v_accvgpr_read_b32 v[vgprValuC+65], acc210 // copy acc to vreg[216] +v_accvgpr_read_b32 v[vgprValuC+70], acc214 // copy acc to vreg[217] +v_accvgpr_read_b32 v[vgprValuC+75], acc218 // copy acc to vreg[218] +v_accvgpr_read_b32 v[vgprValuC+80], acc222 // copy acc to vreg[219] +v_accvgpr_read_b32 v[vgprValuC+85], acc211 // copy acc to vreg[220] +v_accvgpr_read_b32 v[vgprValuC+90], acc215 // copy acc to vreg[221] +v_accvgpr_read_b32 v[vgprValuC+95], acc219 // copy acc to vreg[222] +v_accvgpr_read_b32 v[vgprValuC+100], acc223 // copy acc to vreg[223] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(13, 0, 0, 0), (13, 0, 0, 1), (13, 0, 0, 2), (13, 0, 0, 3), (13, 0, 1, 0), (13, 0, 1, 1), (13, 0, 1, 2), (13, 0, 1, 3), (13, 0, 2, 0), (13, 0, 2, 1), (13, 0, 2, 2), (13, 0, 2, 3), (13, 0, 3, 0), (13, 0, 3, 1), (13, 0, 3, 2), (13, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #14 (d1,d0,vc1,vc0) = */ +/* (14,0,0,0:vw1); (14,0,0,1:vw1); (14,0,0,2:vw1); (14,0,0,3:vw1); (14,0,1,0:vw1); (14,0,1,1:vw1); (14,0,1,2:vw1); (14,0,1,3:vw1); (14,0,2,0:vw1); (14,0,2,1:vw1); (14,0,2,2:vw1); (14,0,2,3:vw1); (14,0,3,0:vw1); (14,0,3,1:vw1); (14,0,3,2:vw1); (14,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(14,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(14,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc224 // copy acc to vreg[224] +v_accvgpr_read_b32 v[vgprValuC+26], acc228 // copy acc to vreg[225] +v_accvgpr_read_b32 v[vgprValuC+33], acc232 // copy acc to vreg[226] +v_accvgpr_read_b32 v[vgprValuC+40], acc236 // copy acc to vreg[227] +v_accvgpr_read_b32 v[vgprValuC+45], acc225 // copy acc to vreg[228] +v_accvgpr_read_b32 v[vgprValuC+50], acc229 // copy acc to vreg[229] +v_accvgpr_read_b32 v[vgprValuC+55], acc233 // copy acc to vreg[230] +v_accvgpr_read_b32 v[vgprValuC+60], acc237 // copy acc to vreg[231] +v_accvgpr_read_b32 v[vgprValuC+65], acc226 // copy acc to vreg[232] +v_accvgpr_read_b32 v[vgprValuC+70], acc230 // copy acc to vreg[233] +v_accvgpr_read_b32 v[vgprValuC+75], acc234 // copy acc to vreg[234] +v_accvgpr_read_b32 v[vgprValuC+80], acc238 // copy acc to vreg[235] +v_accvgpr_read_b32 v[vgprValuC+85], acc227 // copy acc to vreg[236] +v_accvgpr_read_b32 v[vgprValuC+90], acc231 // copy acc to vreg[237] +v_accvgpr_read_b32 v[vgprValuC+95], acc235 // copy acc to vreg[238] +v_accvgpr_read_b32 v[vgprValuC+100], acc239 // copy acc to vreg[239] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(14, 0, 0, 0), (14, 0, 0, 1), (14, 0, 0, 2), (14, 0, 0, 3), (14, 0, 1, 0), (14, 0, 1, 1), (14, 0, 1, 2), (14, 0, 1, 3), (14, 0, 2, 0), (14, 0, 2, 1), (14, 0, 2, 2), (14, 0, 2, 3), (14, 0, 3, 0), (14, 0, 3, 1), (14, 0, 3, 2), (14, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +/* optSingleColVgpr=0 optSharedColVgpr=0 optSGPRUsage=BufferLoad_Edge_Mask optSrdIncForRow=0 biasDim=0 */ + +/******************************************/ +/* Global Write Beta Edge Batch #15 (d1,d0,vc1,vc0) = */ +/* (15,0,0,0:vw1); (15,0,0,1:vw1); (15,0,0,2:vw1); (15,0,0,3:vw1); (15,0,1,0:vw1); (15,0,1,1:vw1); (15,0,1,2:vw1); (15,0,1,3:vw1); (15,0,2,0:vw1); (15,0,2,1:vw1); (15,0,2,2:vw1); (15,0,2,3:vw1); (15,0,3,0:vw1); (15,0,3,1:vw1); (15,0,3,2:vw1); (15,0,3,3:vw1) */ +/******************************************/ + +/* calc coords, apply mask, and issue loads (if necessary) */ +v_mov_b32 v101, BufferOOB +/* (d1,vc1,d0,vc0)=(15,0,0,0) */ +v_add_co_u32 v1, vcc, v1, 13 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +s_mul_i32 s60, s[sgprStrideC1J], 13 // scale stride +v_add_i32 v2, v2, s60 // ROWINC- Move cinRowPtr to next row +s_mul_i32 s60, s[sgprStrideD1J], 13 // scale stride +v_add_i32 v3, v3, s60 // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v13, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v16, v13, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v14, v0, s60 +v_lshlrev_b32 v14, 0x2, v14 // Bias address scaled by BPE +v_cndmask_b32 v14, v101, v14, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v17, v14 offset:0 // load bias +v_lshlrev_b32 v15, 0x2, v0 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v18, v15, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v13, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v13, v101, v13, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v20, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v23, v20, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v21, v4, s60 +v_lshlrev_b32 v21, 0x2, v21 // Bias address scaled by BPE +v_cndmask_b32 v21, v101, v21, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v24, v21 offset:0 // load bias +v_lshlrev_b32 v22, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v25, v22, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v20, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v20, v101, v20, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v27, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v30, v27, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v28, v4, s60 +v_lshlrev_b32 v28, 0x2, v28 // Bias address scaled by BPE +v_cndmask_b32 v28, v101, v28, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v31, v28 offset:0 // load bias +v_lshlrev_b32 v29, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v32, v29, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v27, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v27, v101, v27, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,0,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v34, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v37, v34, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v35, v4, s60 +v_lshlrev_b32 v35, 0x2, v35 // Bias address scaled by BPE +v_cndmask_b32 v35, v101, v35, s[64:65] // LDBias clip if OOB. offset +ds_read_b32 v38, v35 offset:0 // load bias +v_lshlrev_b32 v36, 0x2, v4 // ScaleAlphaVec address scaled by BPE +buffer_load_dword v39, v36, s[sgprSrdScaleAlphaVec:sgprSrdScaleAlphaVec+3], 0 offen offset:0 // load scaleAlphaVecI +v_add_lshl_u32 v34, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v34, v101, v34, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v41, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v44, v41, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v42, v0, s60 +v_lshlrev_b32 v42, 0x2, v42 // Bias address scaled by BPE +v_cndmask_b32 v42, v101, v42, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v43, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v41, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v41, v101, v41, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v46, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v49, v46, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v47, v4, s60 +v_lshlrev_b32 v47, 0x2, v47 // Bias address scaled by BPE +v_cndmask_b32 v47, v101, v47, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v48, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v46, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v46, v101, v46, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v51, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v54, v51, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v52, v4, s60 +v_lshlrev_b32 v52, 0x2, v52 // Bias address scaled by BPE +v_cndmask_b32 v52, v101, v52, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v53, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v51, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v51, v101, v51, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,1,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v56, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v59, v56, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v57, v4, s60 +v_lshlrev_b32 v57, 0x2, v57 // Bias address scaled by BPE +v_cndmask_b32 v57, v101, v57, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v58, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v56, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v56, v101, v56, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v61, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v64, v61, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v62, v0, s60 +v_lshlrev_b32 v62, 0x2, v62 // Bias address scaled by BPE +v_cndmask_b32 v62, v101, v62, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v63, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v61, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v61, v101, v61, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v66, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v69, v66, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v67, v4, s60 +v_lshlrev_b32 v67, 0x2, v67 // Bias address scaled by BPE +v_cndmask_b32 v67, v101, v67, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v68, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v66, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v66, v101, v66, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v71, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v74, v71, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v72, v4, s60 +v_lshlrev_b32 v72, 0x2, v72 // Bias address scaled by BPE +v_cndmask_b32 v72, v101, v72, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v73, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v71, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v71, v101, v71, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,2,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v76, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v79, v76, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v77, v4, s60 +v_lshlrev_b32 v77, 0x2, v77 // Bias address scaled by BPE +v_cndmask_b32 v77, v101, v77, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v78, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v76, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v76, v101, v76, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,0) */ +v_add_co_u32 v1, vcc, v1, 1 // coord1.1: coord1Vgpr += d1*sg1*VW + vc1 + +/* Fix for UseInitialStridesCD, emitAddressSetupCode */ +v_add_u32 v2, v2, s[sgprStrideC1J] // ROWINC- Move cinRowPtr to next row +v_add_u32 v3, v3, s[sgprStrideD1J] // Move coutRowPtrD to next row +v_cmp_lt_u32 s[60:61], v0, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v81, v2, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v84, v81, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v82, v0, s60 +v_lshlrev_b32 v82, 0x2, v82 // Bias address scaled by BPE +v_cndmask_b32 v82, v101, v82, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v83, 0x2, v0 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v81, v3, v0, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v81, v101, v81, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,1) */ +v_add_co_u32 v4, vcc, v0, 1 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v86, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v89, v86, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v87, v4, s60 +v_lshlrev_b32 v87, 0x2, v87 // Bias address scaled by BPE +v_cndmask_b32 v87, v101, v87, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v88, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v86, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v86, v101, v86, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,2) */ +v_add_co_u32 v4, vcc, v0, 2 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v91, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v94, v91, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v92, v4, s60 +v_lshlrev_b32 v92, 0x2, v92 // Bias address scaled by BPE +v_cndmask_b32 v92, v101, v92, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v93, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v91, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v91, v101, v91, s[64:65] // LDD clip if OOB. offset +/* (d1,vc1,d0,vc0)=(15,3,0,3) */ +v_add_co_u32 v4, vcc, v0, 3 // coord0.1: coord0 += d0*sg0*VW + vc0 +v_cmp_lt_u32 s[60:61], v4, s[sgprSizeI] // coord0 < size0 +v_cmp_lt_u32 s[64:65], v1, s[sgprSizeJ] // coord1 < size1 +s_and_b64 s[64:65], s[60:61], s[64:65] // in0 && in1 +v_add_lshl_u32 v96, v2, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDC clip if OOB. offset +buffer_load_short_d16 v99, v96, s[sgprSrdC:sgprSrdC+3], 0 offen offset:0 // load C +s_mul_i32 s60, 256, s[sgprWorkGroup0] // wgp0 * MT0 +v_sub_u32 v97, v4, s60 +v_lshlrev_b32 v97, 0x2, v97 // Bias address scaled by BPE +v_cndmask_b32 v97, v101, v97, s[64:65] // LDBias clip if OOB. offset +v_lshlrev_b32 v98, 0x2, v4 // ScaleAlphaVec address scaled by BPE +v_add_lshl_u32 v96, v3, v4, 0x1 // scaleToBpe: accumulate d0 lower and *= bpe into Cin addr +v_cndmask_b32 v96, v101, v96, s[64:65] // LDD clip if OOB. offset +v_accvgpr_read_b32 v[vgprValuC+19], acc240 // copy acc to vreg[240] +v_accvgpr_read_b32 v[vgprValuC+26], acc244 // copy acc to vreg[241] +v_accvgpr_read_b32 v[vgprValuC+33], acc248 // copy acc to vreg[242] +v_accvgpr_read_b32 v[vgprValuC+40], acc252 // copy acc to vreg[243] +v_accvgpr_read_b32 v[vgprValuC+45], acc241 // copy acc to vreg[244] +v_accvgpr_read_b32 v[vgprValuC+50], acc245 // copy acc to vreg[245] +v_accvgpr_read_b32 v[vgprValuC+55], acc249 // copy acc to vreg[246] +v_accvgpr_read_b32 v[vgprValuC+60], acc253 // copy acc to vreg[247] +v_accvgpr_read_b32 v[vgprValuC+65], acc242 // copy acc to vreg[248] +v_accvgpr_read_b32 v[vgprValuC+70], acc246 // copy acc to vreg[249] +v_accvgpr_read_b32 v[vgprValuC+75], acc250 // copy acc to vreg[250] +v_accvgpr_read_b32 v[vgprValuC+80], acc254 // copy acc to vreg[251] +v_accvgpr_read_b32 v[vgprValuC+85], acc243 // copy acc to vreg[252] +v_accvgpr_read_b32 v[vgprValuC+90], acc247 // copy acc to vreg[253] +v_accvgpr_read_b32 v[vgprValuC+95], acc251 // copy acc to vreg[254] +v_accvgpr_read_b32 v[vgprValuC+100], acc255 // copy acc to vreg[255] +s_nop 1 // 2 wait states required before reading vgpr + +/* rC *= alpha batchElements=[(15, 0, 0, 0), (15, 0, 0, 1), (15, 0, 0, 2), (15, 0, 0, 3), (15, 0, 1, 0), (15, 0, 1, 1), (15, 0, 1, 2), (15, 0, 1, 3), (15, 0, 2, 0), (15, 0, 2, 1), (15, 0, 2, 2), (15, 0, 2, 3), (15, 0, 3, 0), (15, 0, 3, 1), (15, 0, 3, 2), (15, 0, 3, 3)] */ +v_mul_f32 v[vgprValuC+19], s[sgprAlpha], v[vgprValuC+19] // *= alpha +v_mul_f32 v[vgprValuC+26], s[sgprAlpha], v[vgprValuC+26] // *= alpha +v_mul_f32 v[vgprValuC+33], s[sgprAlpha], v[vgprValuC+33] // *= alpha +v_mul_f32 v[vgprValuC+40], s[sgprAlpha], v[vgprValuC+40] // *= alpha +v_mul_f32 v[vgprValuC+45], s[sgprAlpha], v[vgprValuC+45] // *= alpha +v_mul_f32 v[vgprValuC+50], s[sgprAlpha], v[vgprValuC+50] // *= alpha +v_mul_f32 v[vgprValuC+55], s[sgprAlpha], v[vgprValuC+55] // *= alpha +v_mul_f32 v[vgprValuC+60], s[sgprAlpha], v[vgprValuC+60] // *= alpha +v_mul_f32 v[vgprValuC+65], s[sgprAlpha], v[vgprValuC+65] // *= alpha +v_mul_f32 v[vgprValuC+70], s[sgprAlpha], v[vgprValuC+70] // *= alpha +v_mul_f32 v[vgprValuC+75], s[sgprAlpha], v[vgprValuC+75] // *= alpha +v_mul_f32 v[vgprValuC+80], s[sgprAlpha], v[vgprValuC+80] // *= alpha +v_mul_f32 v[vgprValuC+85], s[sgprAlpha], v[vgprValuC+85] // *= alpha +v_mul_f32 v[vgprValuC+90], s[sgprAlpha], v[vgprValuC+90] // *= alpha +v_mul_f32 v[vgprValuC+95], s[sgprAlpha], v[vgprValuC+95] // *= alpha +v_mul_f32 v[vgprValuC+100], s[sgprAlpha], v[vgprValuC+100] // *= alpha +s_waitcnt 0 // wait for Beta, ScaleAlphaVec, Bias LDS + +/* apply mask, calc new C and issue writes */ +v_mov_b32 v10, 0xffff0000 // mask for pack two bfloat16 element to 32bit +v_mov_b32 v11, 0x7fff0000 // fp32 Nan +v_mov_b32 v12, 0x7fff // rounding bias for bfloat16 +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+19], v18, v[vgprValuC+19] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v16 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+19], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+19] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v19, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+19], v[vgprValuC+19] // check Nan +v_bfe_u32 v9, v[vgprValuC+19], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+19], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+19], v9, v11, s[60:61] +v_lshrrev_b32 v19, 16, v[vgprValuC+19] // convert C to bf16 +buffer_store_short v19, v13, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+26], v25, v[vgprValuC+26] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v23 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+26], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+26] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v26, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+26], v[vgprValuC+26] // check Nan +v_bfe_u32 v9, v[vgprValuC+26], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+26], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+26], v9, v11, s[60:61] +v_lshrrev_b32 v26, 16, v[vgprValuC+26] // convert C to bf16 +buffer_store_short v26, v20, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+33], v32, v[vgprValuC+33] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v30 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+33], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+33] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v33, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+33], v[vgprValuC+33] // check Nan +v_bfe_u32 v9, v[vgprValuC+33], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+33], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+33], v9, v11, s[60:61] +v_lshrrev_b32 v33, 16, v[vgprValuC+33] // convert C to bf16 +buffer_store_short v33, v27, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+40], v39, v[vgprValuC+40] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v37 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+40], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+40] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v40, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+40], v[vgprValuC+40] // check Nan +v_bfe_u32 v9, v[vgprValuC+40], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+40], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+40], v9, v11, s[60:61] +v_lshrrev_b32 v40, 16, v[vgprValuC+40] // convert C to bf16 +buffer_store_short v40, v34, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+45], v18, v[vgprValuC+45] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v44 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+45], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+45] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v45, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+45], v[vgprValuC+45] // check Nan +v_bfe_u32 v9, v[vgprValuC+45], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+45], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+45], v9, v11, s[60:61] +v_lshrrev_b32 v45, 16, v[vgprValuC+45] // convert C to bf16 +buffer_store_short v45, v41, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+50], v25, v[vgprValuC+50] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v49 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+50], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+50] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v50, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+50], v[vgprValuC+50] // check Nan +v_bfe_u32 v9, v[vgprValuC+50], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+50], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+50], v9, v11, s[60:61] +v_lshrrev_b32 v50, 16, v[vgprValuC+50] // convert C to bf16 +buffer_store_short v50, v46, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+55], v32, v[vgprValuC+55] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v54 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+55], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+55] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v55, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+55], v[vgprValuC+55] // check Nan +v_bfe_u32 v9, v[vgprValuC+55], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+55], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+55], v9, v11, s[60:61] +v_lshrrev_b32 v55, 16, v[vgprValuC+55] // convert C to bf16 +buffer_store_short v55, v51, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+60], v39, v[vgprValuC+60] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v59 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+60], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+60] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v60, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+60], v[vgprValuC+60] // check Nan +v_bfe_u32 v9, v[vgprValuC+60], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+60], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+60], v9, v11, s[60:61] +v_lshrrev_b32 v60, 16, v[vgprValuC+60] // convert C to bf16 +buffer_store_short v60, v56, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+65], v18, v[vgprValuC+65] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v64 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+65], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+65] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v65, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+65], v[vgprValuC+65] // check Nan +v_bfe_u32 v9, v[vgprValuC+65], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+65], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+65], v9, v11, s[60:61] +v_lshrrev_b32 v65, 16, v[vgprValuC+65] // convert C to bf16 +buffer_store_short v65, v61, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+70], v25, v[vgprValuC+70] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v69 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+70], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+70] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v70, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+70], v[vgprValuC+70] // check Nan +v_bfe_u32 v9, v[vgprValuC+70], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+70], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+70], v9, v11, s[60:61] +v_lshrrev_b32 v70, 16, v[vgprValuC+70] // convert C to bf16 +buffer_store_short v70, v66, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+75], v32, v[vgprValuC+75] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v74 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+75], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+75] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v75, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+75], v[vgprValuC+75] // check Nan +v_bfe_u32 v9, v[vgprValuC+75], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+75], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+75], v9, v11, s[60:61] +v_lshrrev_b32 v75, 16, v[vgprValuC+75] // convert C to bf16 +buffer_store_short v75, v71, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+80], v39, v[vgprValuC+80] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v79 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+80], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+80] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v80, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+80], v[vgprValuC+80] // check Nan +v_bfe_u32 v9, v[vgprValuC+80], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+80], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+80], v9, v11, s[60:61] +v_lshrrev_b32 v80, 16, v[vgprValuC+80] // convert C to bf16 +buffer_store_short v80, v76, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v18, 1.0, v18, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+85], v18, v[vgprValuC+85] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v84 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+85], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v17, v[vgprValuC+85] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v85, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+85], v[vgprValuC+85] // check Nan +v_bfe_u32 v9, v[vgprValuC+85], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+85], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+85], v9, v11, s[60:61] +v_lshrrev_b32 v85, 16, v[vgprValuC+85] // convert C to bf16 +buffer_store_short v85, v81, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v25, 1.0, v25, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+90], v25, v[vgprValuC+90] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v89 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+90], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v24, v[vgprValuC+90] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v90, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+90], v[vgprValuC+90] // check Nan +v_bfe_u32 v9, v[vgprValuC+90], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+90], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+90], v9, v11, s[60:61] +v_lshrrev_b32 v90, 16, v[vgprValuC+90] // convert C to bf16 +buffer_store_short v90, v86, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v32, 1.0, v32, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+95], v32, v[vgprValuC+95] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v94 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+95], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v31, v[vgprValuC+95] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v95, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+95], v[vgprValuC+95] // check Nan +v_bfe_u32 v9, v[vgprValuC+95], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+95], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+95], v9, v11, s[60:61] +v_lshrrev_b32 v95, 16, v[vgprValuC+95] // convert C to bf16 +buffer_store_short v95, v91, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +v_cmp_gt_u32 s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1], s[sgprSrdScaleAlphaVec+2], 0 // == 0 ? +v_cndmask_b32 v39, 1.0, v39, s[sgprAddressScaleAlphaVec:sgprAddressScaleAlphaVec+1] // 1. mul 1 if 0 +v_mul_f32 v[vgprValuC+100], v39, v[vgprValuC+100] // *= scaleAlphaVecVMul +v_lshlrev_b32 v4, 16, v99 // cvt bf16 to fp32. +v_fmac_f32 v[vgprValuC+100], v4, s[sgprBeta] // finalSum = sum*alpha + C*beta +v_add_f32 v4, v38, v[vgprValuC+100] // C += bias +s_swappc_b64 s[58:59], s[12:13] +v_mov_b32 v100, v4 +v_cmp_u_f32 s[60:61], v[vgprValuC+100], v[vgprValuC+100] // check Nan +v_bfe_u32 v9, v[vgprValuC+100], 16, 1 // Non-Nan case: store lsb of bf16 +v_add3_u32 v9, v[vgprValuC+100], v9, v12 // Non-Nan case: add lsb and the increment for rounding +v_cndmask_b32 v[vgprValuC+100], v9, v11, s[60:61] +v_lshrrev_b32 v100, 16, v[vgprValuC+100] // convert C to bf16 +buffer_store_short v100, v96, s[sgprSrdD:sgprSrdD+3], 0 offen offset:0 // store D +s_nop 0 // 1 wait state required when next inst writes vgprs held by previous dwordx4 store inst +s_branch label_GW_End_2 // jump to end +label_Activation_None_VW1: +s_setpc_b64 s[58:59] +label_Activation_Abs_VW1: +v_and_b32 v4, 0x7fffffff, v4 // Remove sign bit +s_setpc_b64 s[58:59] +label_Activation_Clippedrelu_VW1: +v_cmp_gt_f32 vcc, v4, s[sgpractivationAlpha] // x > alpha ? +v_min_f32 v4, s[sgpractivationBeta], v4 // min(x, beta) +v_cndmask_b32 v4, 0.0, v4, vcc // set x to 0 if <= alpha +s_setpc_b64 s[58:59] +label_Activation_Gelu_VW1: +v_mul_f32 v8, 0x3d372713, v4 // k1 * x +v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) +v_mul_f32 v4, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +s_setpc_b64 s[58:59] +label_Activation_Leakyrelu_VW1: +v_mul_f32 v8, s[sgpractivationAlpha], v4 // tmp = x * alpha +v_cmp_ge_f32 vcc, v4, 0.0 // x >= 0 ? +v_cndmask_b32 v4, v8, v4, vcc // set x to tmp if < 0 +s_setpc_b64 s[58:59] +label_Activation_Relu_VW1: +v_max_f32 v4, v4, 0 // x = max(0, x) +s_setpc_b64 s[58:59] +label_Activation_Sigmoid_VW1: +v_mul_f32 v4, 0xbfb8aa3b, v4 // (fused -1.442695) +v_exp_f32 v4, v4 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v4, 1.0, v4 // 1 + exp(-x) +v_rcp_f32 v4, v4 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +s_setpc_b64 s[58:59] +label_Activation_Tanh_VW1: +v_mul_f32 v4, s[sgpractivationAlpha], v4 // x * alpha +v_mul_f32 v4, 0x4038aa3b, v4 // (fused 2) +v_exp_f32 v4, v4 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v4, 1.0, v4 // e^2x + 1 +v_rcp_f32 v4, v4 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v4, -2.0, v4, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 +v_mul_f32 v4, s[sgpractivationBeta], v4 // beta * tanh(x) +s_setpc_b64 s[58:59] +label_Activation_Geluscaling_VW1: +v_mul_f32 v8, 0x3d372713, v4 // k1 * x +v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) +v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v4, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale +s_setpc_b64 s[58:59] +label_Activation_Silu_VW1: +v_mul_f32 v8, -1.4426950408889634, v4 // (fused -1.442695) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // 1 + exp(-x) +v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v4, v4, v8 // x / (1 + exp(-x)) +s_setpc_b64 s[58:59] +label_GW_End_2: +label_KernelEnd: +s_endpgm // Kernel End +label_Activation_None_VW4: +s_setpc_b64 s[58:59] +label_Activation_Abs_VW4: +v_and_b32 v4, 0x7fffffff, v4 // Remove sign bit +v_and_b32 v5, 0x7fffffff, v5 // Remove sign bit +v_and_b32 v6, 0x7fffffff, v6 // Remove sign bit +v_and_b32 v7, 0x7fffffff, v7 // Remove sign bit +s_setpc_b64 s[58:59] +label_Activation_Clippedrelu_VW4: +v_cmp_gt_f32 vcc, v4, s[sgpractivationAlpha] // x > alpha ? +v_min_f32 v4, s[sgpractivationBeta], v4 // min(x, beta) +v_cndmask_b32 v4, 0.0, v4, vcc // set x to 0 if <= alpha +v_cmp_gt_f32 vcc, v5, s[sgpractivationAlpha] // x > alpha ? +v_min_f32 v5, s[sgpractivationBeta], v5 // min(x, beta) +v_cndmask_b32 v5, 0.0, v5, vcc // set x to 0 if <= alpha +v_cmp_gt_f32 vcc, v6, s[sgpractivationAlpha] // x > alpha ? +v_min_f32 v6, s[sgpractivationBeta], v6 // min(x, beta) +v_cndmask_b32 v6, 0.0, v6, vcc // set x to 0 if <= alpha +v_cmp_gt_f32 vcc, v7, s[sgpractivationAlpha] // x > alpha ? +v_min_f32 v7, s[sgpractivationBeta], v7 // min(x, beta) +v_cndmask_b32 v7, 0.0, v7, vcc // set x to 0 if <= alpha +s_setpc_b64 s[58:59] +label_Activation_Gelu_VW4: +v_mul_f32 v8, 0x3d372713, v4 // k1 * x +v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) +v_mul_f32 v4, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v8, 0x3d372713, v5 // k1 * x +v_fma_f32 v8, v5, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v5, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v5, v8 // x * (1 + tanh(...)) +v_mul_f32 v5, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v8, 0x3d372713, v6 // k1 * x +v_fma_f32 v8, v6, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v6, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v6, v8 // x * (1 + tanh(...)) +v_mul_f32 v6, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v8, 0x3d372713, v7 // k1 * x +v_fma_f32 v8, v7, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v7, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v7, v8 // x * (1 + tanh(...)) +v_mul_f32 v7, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +s_setpc_b64 s[58:59] +label_Activation_Leakyrelu_VW4: +v_mul_f32 v8, s[sgpractivationAlpha], v4 // tmp = x * alpha +v_cmp_ge_f32 vcc, v4, 0.0 // x >= 0 ? +v_cndmask_b32 v4, v8, v4, vcc // set x to tmp if < 0 +v_mul_f32 v8, s[sgpractivationAlpha], v5 // tmp = x * alpha +v_cmp_ge_f32 vcc, v5, 0.0 // x >= 0 ? +v_cndmask_b32 v5, v8, v5, vcc // set x to tmp if < 0 +v_mul_f32 v8, s[sgpractivationAlpha], v6 // tmp = x * alpha +v_cmp_ge_f32 vcc, v6, 0.0 // x >= 0 ? +v_cndmask_b32 v6, v8, v6, vcc // set x to tmp if < 0 +v_mul_f32 v8, s[sgpractivationAlpha], v7 // tmp = x * alpha +v_cmp_ge_f32 vcc, v7, 0.0 // x >= 0 ? +v_cndmask_b32 v7, v8, v7, vcc // set x to tmp if < 0 +s_setpc_b64 s[58:59] +label_Activation_Relu_VW4: +v_max_f32 v4, v4, 0 // x = max(0, x) +v_max_f32 v5, v5, 0 // x = max(0, x) +v_max_f32 v6, v6, 0 // x = max(0, x) +v_max_f32 v7, v7, 0 // x = max(0, x) +s_setpc_b64 s[58:59] +label_Activation_Sigmoid_VW4: +v_mul_f32 v4, 0xbfb8aa3b, v4 // (fused -1.442695) +v_exp_f32 v4, v4 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v4, 1.0, v4 // 1 + exp(-x) +v_rcp_f32 v4, v4 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v5, 0xbfb8aa3b, v5 // (fused -1.442695) +v_exp_f32 v5, v5 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v5, 1.0, v5 // 1 + exp(-x) +v_rcp_f32 v5, v5 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v6, 0xbfb8aa3b, v6 // (fused -1.442695) +v_exp_f32 v6, v6 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v6, 1.0, v6 // 1 + exp(-x) +v_rcp_f32 v6, v6 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v7, 0xbfb8aa3b, v7 // (fused -1.442695) +v_exp_f32 v7, v7 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v7, 1.0, v7 // 1 + exp(-x) +v_rcp_f32 v7, v7 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +s_setpc_b64 s[58:59] +label_Activation_Tanh_VW4: +v_mul_f32 v4, s[sgpractivationAlpha], v4 // x * alpha +v_mul_f32 v4, 0x4038aa3b, v4 // (fused 2) +v_exp_f32 v4, v4 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v4, 1.0, v4 // e^2x + 1 +v_rcp_f32 v4, v4 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v4, -2.0, v4, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 +v_mul_f32 v4, s[sgpractivationBeta], v4 // beta * tanh(x) +v_mul_f32 v5, s[sgpractivationAlpha], v5 // x * alpha +v_mul_f32 v5, 0x4038aa3b, v5 // (fused 2) +v_exp_f32 v5, v5 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v5, 1.0, v5 // e^2x + 1 +v_rcp_f32 v5, v5 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v5, -2.0, v5, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 +v_mul_f32 v5, s[sgpractivationBeta], v5 // beta * tanh(x) +v_mul_f32 v6, s[sgpractivationAlpha], v6 // x * alpha +v_mul_f32 v6, 0x4038aa3b, v6 // (fused 2) +v_exp_f32 v6, v6 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v6, 1.0, v6 // e^2x + 1 +v_rcp_f32 v6, v6 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v6, -2.0, v6, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 +v_mul_f32 v6, s[sgpractivationBeta], v6 // beta * tanh(x) +v_mul_f32 v7, s[sgpractivationAlpha], v7 // x * alpha +v_mul_f32 v7, 0x4038aa3b, v7 // (fused 2) +v_exp_f32 v7, v7 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v7, 1.0, v7 // e^2x + 1 +v_rcp_f32 v7, v7 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v7, -2.0, v7, 1.0 // (-2) * (1 / (e^2x + 1)) + 1 +v_mul_f32 v7, s[sgpractivationBeta], v7 // beta * tanh(x) +s_setpc_b64 s[58:59] +label_Activation_Geluscaling_VW4: +v_mul_f32 v8, 0x3d372713, v4 // k1 * x +v_fma_f32 v8, v4, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v4, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v4, v8 // x * (1 + tanh(...)) +v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v4, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale +v_mul_f32 v8, 0x3d372713, v5 // k1 * x +v_fma_f32 v8, v5, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v5, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v5, v8 // x * (1 + tanh(...)) +v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v5, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale +v_mul_f32 v8, 0x3d372713, v6 // k1 * x +v_fma_f32 v8, v6, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v6, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v6, v8 // x * (1 + tanh(...)) +v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v6, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale +v_mul_f32 v8, 0x3d372713, v7 // k1 * x +v_fma_f32 v8, v7, v8, 1.0 // 1 + (k1 * x * x) +v_mul_f32 v8, v7, v8 // x * (1 + k1 * x * x) +v_mul_f32 v8, 0x40135761, v8 // (fused 2.302208) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // e^2x + 1 +v_rcp_f32 v8, v8 // 1 / (e^2x + 1) +s_nop 0 // 1 wait states +v_fma_f32 v8, -2.0, v8, 2.0 // ( + 1 (fused)) +v_mul_f32 v8, v7, v8 // x * (1 + tanh(...)) +v_mul_f32 v8, 0.5, v8 // 0.5 * x * (1 + tanh(...)) +v_mul_f32 v7, s[sgpractivationAlpha], v8 // 0.5 * x * (1 + tanh(...)) * scale +s_setpc_b64 s[58:59] +label_Activation_Silu_VW4: +v_mul_f32 v8, -1.4426950408889634, v4 // (fused -1.442695) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // 1 + exp(-x) +v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v4, v4, v8 // x / (1 + exp(-x)) +v_mul_f32 v8, -1.4426950408889634, v5 // (fused -1.442695) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // 1 + exp(-x) +v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v5, v5, v8 // x / (1 + exp(-x)) +v_mul_f32 v8, -1.4426950408889634, v6 // (fused -1.442695) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // 1 + exp(-x) +v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v6, v6, v8 // x / (1 + exp(-x)) +v_mul_f32 v8, -1.4426950408889634, v7 // (fused -1.442695) +v_exp_f32 v8, v8 // exp step 2 +s_nop 0 // 1 wait states +v_add_f32 v8, 1.0, v8 // 1 + exp(-x) +v_rcp_f32 v8, v8 // 1 / (1 + exp(-x)) +s_nop 0 // 1 wait states +v_mul_f32 v7, v7, v8 // x / (1 + exp(-x)) +s_setpc_b64 s[58:59] +s_endpgm +label_ASM_End: /// The end of the kernel diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s index 4f9a429c85..6d9369b256 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_BBS_BH_Bias_AS_SAV_UserArgs_MT256x256x64_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s @@ -61,7 +61,27 @@ custom.config: Batched: True GroupedGemm: False SupportUserArgs: True - MatrixInstruction: [16, 16, 16, 1, 1, 4, 16, 4, 1] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [4, 16] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 64 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_DTVA.s index 566eb28e47..ebb87bef67 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_DTVA.s @@ -64,6 +64,7 @@ custom.config: GroupedGemm: False SupportUserArgs: True MatrixInstruction: [16, 16, 32, 1, 1, 4, 14, 4, 1] + WavefrontSize: 64 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 128 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s index cfc8b39984..225e125618 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x224x128_MI16x16x1_SN_K1_MIWT4_14_WSGRB2_DTVA.s @@ -64,6 +64,7 @@ custom.config: GroupedGemm: False SupportUserArgs: True MatrixInstruction: [16, 16, 32, 1, 1, 4, 14, 4, 1] + WavefrontSize: 64 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 128 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_DTVA.s index 9732eedb6c..862f85074c 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_DTVA.s @@ -64,6 +64,7 @@ custom.config: GroupedGemm: False SupportUserArgs: True MatrixInstruction: [16, 16, 32, 1, 1, 4, 16, 4, 1] + WavefrontSize: 64 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 128 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s index f4f20252a4..b156ac1179 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_F8NBS_BH_BiasSB_AS_SABV_SAV_UserArgs_MT256x256x128_MI16x16x1_SN_K1_MIWT4_16_WSGRB2_DTVA.s @@ -64,6 +64,7 @@ custom.config: GroupedGemm: False SupportUserArgs: True MatrixInstruction: [16, 16, 32, 1, 1, 4, 16, 4, 1] + WavefrontSize: 64 1LDSBuffer: 1 ScheduleIterAlg: 3 DepthU: 128 diff --git a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_HHS_BH_UserArgs_MT128x16x128_MI16x16x1_SN_K1_MIWT2_1_triple_buffer.s b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_HHS_BH_UserArgs_MT128x16x128_MI16x16x1_SN_K1_MIWT2_1_triple_buffer.s index 586ab4fbc1..8ff3e883fa 100644 --- a/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_HHS_BH_UserArgs_MT128x16x128_MI16x16x1_SN_K1_MIWT2_1_triple_buffer.s +++ b/tensilelite/Tensile/CustomKernels/Custom_Cijk_Alik_Bljk_HHS_BH_UserArgs_MT128x16x128_MI16x16x1_SN_K1_MIWT2_1_triple_buffer.s @@ -56,7 +56,27 @@ custom.config: TransposeB: 0 UseBeta: True Batched: True - MatrixInstruction: [16, 16,16, 1, 1, 2,1, 4,1 ] + EnableF32XdlMathOp: False + EnableMatrixInstruction: True + MFMA_BF16_1K: False + MIBlock: [16, 16, 16, 1, 1, 1] + MIInputPerThread: 4 + MIInputPerThreadA: 4 + MIInputPerThreadB: 4 + MIInputPerThreadMetadata: 4 + MIWaveGroup: [4, 1] + MIWaveTile: [2, 1] + MatrixInstB: 1 + MatrixInstBM: 1 + MatrixInstBN: 1 + MatrixInstK: 16 + MatrixInstM: 16 + MatrixInstN: 16 + MatrixInstruction: [16, 16, 16, 1] + Sparse: 0 + ThreadTile: [1, 1] + WavefrontSize: 64 + WorkGroup: [64, 4, 1] AssertFree0ElementMultiple: 16 AssertFree1ElementMultiple: 1 AssertSummationElementMultiple: 128 diff --git a/tensilelite/Tensile/GenerateSummations.py b/tensilelite/Tensile/GenerateSummations.py index 3b791d1c92..65389a454b 100644 --- a/tensilelite/Tensile/GenerateSummations.py +++ b/tensilelite/Tensile/GenerateSummations.py @@ -37,7 +37,7 @@ from . import ClientWriter from .Common import assignGlobalParameters, ensurePath, globalParameters, \ - printExit, isaToGfx, gfxToSwCodename + printExit, isaToGfx, gfxToSwCodename, detectGlobalCurrentISA from .SolutionStructs import ProblemSizes from .Toolchain.Validators import ToolchainDefaults, validateToolchain @@ -64,10 +64,10 @@ def GenerateSummations(userArgs): inputLogicPath = userArgs[0] outputPath = userArgs[1] - assignGlobalParameters({}) + isaInfoMap = assignGlobalParameters({}) cxxCompiler, cCompiler = validateToolchain(ToolchainDefaults.CXX_COMPILER, ToolchainDefaults.C_COMPILER) - currentISA = globalParameters["CurrentISA"] + currentISA = detectGlobalCurrentISA(0) gfxName = isaToGfx(currentISA) commonName = gfxToSwCodename(gfxName) @@ -93,7 +93,7 @@ def GenerateSummations(userArgs): # same as the initial logic with the summation model added. To preseve the original # logic we also read in the raw unaltered version of the logic and stage the content # to write the final logic. - logic = LibraryIO.parseLibraryLogicFile(logicFileName, cxxCompiler) + logic = LibraryIO.parseLibraryLogicFile(logicFileName, cxxCompiler, isaInfoMap) rawLogic = LibraryIO.rawLibraryLogic(logicFileName) # If we cannot read the logic file then skip it diff --git a/tensilelite/Tensile/KernelWriter.py b/tensilelite/Tensile/KernelWriter.py index 2daacd4263..1c7abd0651 100644 --- a/tensilelite/Tensile/KernelWriter.py +++ b/tensilelite/Tensile/KernelWriter.py @@ -32,12 +32,13 @@ from .TensilePass import TensilePass, TensilePassOptions from .Component import Component, LraTileProperties from .Components.Signature import UserArgumentsInfo -from .CustomKernels import isCustomKernelConfig from .SolutionStructs import Solution, isPackedIndex from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationModule -from .Common import globalParameters, printWarning, roundUp, print2, printExit, DataDirection, SemanticVersion, \ - INDEX_CHARS, MAX_FILENAME_LENGTH +from .Common import printWarning, roundUp, print2, DebugConfig, DataDirection, \ + INDEX_CHARS, IsaVersion +from Tensile.SolutionStructs.Naming import getKernelName +from Tensile.Toolchain.Component import Assembler import abc import os @@ -349,6 +350,7 @@ class ExternClasses: activation: ActivationModule = ActivationModule() biasSumUnroll: Optional[Component.SumUnroll] = None + ################################################################################ # Kernel Writer ################################################################################ @@ -358,12 +360,18 @@ class KernelWriter(metaclass=abc.ABCMeta): ############################################################################## # Init ############################################################################## - def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion): + def __init__( + self, + kernelMinNaming, + kernelSerialNaming, + assembler: Assembler, + debugConfig: DebugConfig, + ): self.kernelMinNaming = kernelMinNaming self.kernelSerialNaming = kernelSerialNaming self.assembler = assembler - self.amdClangVersion = amdClangVersion self.ti = None + self.debugConfig = debugConfig self.do = {} self.do["PreLoop"] = True @@ -393,7 +401,7 @@ def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClang # Various debug flags and modes self.db = {} - self.db["EnableAsserts"] = globalParameters["EnableAsserts"] # Enable assertion codegen. Requires 2 SGPR. + self.db["EnableAsserts"] = self.debugConfig.enableAsserts # Enable assertion codegen. Requires 2 SGPR. self.db["DebugKernelMaxItems"] = 16 # Capture first N(=16) print values, ignore subsequent. If -1, debug writing is faster but writing more than 16 values is undefined. # Chicken bit to add conservative synchronization at strategic points: @@ -420,8 +428,8 @@ def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClang # Requires DataInitTypeAB=1. # Only works if the problem uses full tiles (no edges) # Mismatches will assert (generate GPUVM fault) - self.db["CheckValue1A"] = globalParameters["EnableDebugA"] - self.db["CheckValue1B"] = globalParameters["EnableDebugB"] + self.db["CheckValue1A"] = self.debugConfig.enableDebugA + self.db["CheckValue1B"] = self.debugConfig.enableDebugB self.db["CheckValue1Metadata"] = False # Check value in C matrix. # Caveats: @@ -430,15 +438,15 @@ def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClang # - Only works if matrix is integral multiple of macro-tile (no edges) - check is dumb so doesn't know # which work-items are outside the valid edge. # - Does not work in OptNoLoadLoop - self.db["CheckValueC"] = globalParameters["EnableDebugC"] + self.db["CheckValueC"] = self.debugConfig.enableDebugC # value expected if CheckValueC is set. Use '.' for FP. # For example could be 16.0 if U=8 and alpha=2 - self.db["ValueCExpectedValue"] = globalParameters["ExpectedValueC"] + self.db["ValueCExpectedValue"] = self.debugConfig.expectedValueC # Force an expected value for all C outputs. # May be useful for checking store path # See same caveats as CheckValueC - self.db["ForceExpectedValue"] = globalParameters["ForceCExpectedValue"] + self.db["ForceExpectedValue"] = self.debugConfig.forceCExpectedValue # Force VSerial value into the output, this will # not match reference but can be useful to see which work-items are @@ -468,7 +476,7 @@ def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClang self.db["PrintStoreRegisterDb"] = False self.dumpData = Dump("DebugKernelItems", "AddressDbg", self.db["DebugKernelMaxItems"], \ - globalParameters["DebugKernel"]) + self.debugConfig.debugKernel) self.labels = LabelManager() # KernelWriter values @@ -560,7 +568,7 @@ def makeSchedule(self, kernel, tensorParametersA, tensorParametersB, localWriteE # returns: a Module with the combined, optimally scheduled # localReadCode + otherCode ############################################################################## - def makeSubIterSchedule(self, kernel, tPA, tPB, localReadCode, iteration, pointerLWCode, pointerLRCode, waitCode, macIterCode, \ + def _makeSubIterSchedule(self, kernel, tPA, tPB, localReadCode, iteration, pointerLWCode, pointerLRCode, waitCode, macIterCode, \ waitLWCode = Module(), syncCode = Module(), packCode = Module(), prevIterCode = Module(), NLLlast = False): iterCode = Module() @@ -1449,17 +1457,12 @@ def hasAnyDependency(lr: DSLoadInstruction, insts: List[Instruction]): # (which is always just before the macs) lgkmcnt += localWrites else: - # if UnrollLoopEfficiencyEnable == True use waitCode passed lgkmCnt - # else: # we need to wait for all preceding reads before the macs # so only opportunity for optimization is if the writes are at the end - if globalParameters["UnrollLoopEfficiencyEnable"]: - lgkmcnt = waitCode.lgkmcnt + if localReads: + lgkmcnt = 0 # reset to wait for all reads else: - if localReads: - lgkmcnt = 0 # reset to wait for all reads - else: - lgkmcnt = localWrites # this only survives if writes are at the end + lgkmcnt = localWrites # this only survives if writes are at the end waitCode.comment += " old=%u, new=%u newLW=%u newLR=%u" % (waitCode.lgkmcnt, lgkmcnt,localWrites,localReads) if iteration == 0: @@ -1922,7 +1925,7 @@ def noLoadLoopBody( self, kernel, tensorParametersA, tensorParametersB, pack, is tP = tensorParametersA if kernel["ProblemType"]["BiasSrc"] == "A" else tensorParametersB macIterCode.add(self.exclasses.biasSumUnroll.loopSum(self, kernel, tP, u, kernel["InnerUnroll"])) - subIterCode = self.makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ + subIterCode = self._makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ u, pointerLWCode, pointerLRCode, waitCode, macIterCode, waitLWCode, syncCode, pack[luIdx], module, NLLlast) module.add(subIterCode) pack[luIdx] = Module() @@ -2005,7 +2008,7 @@ def noLoadLoop( self, kernel, tensorParametersA, tensorParametersB, isOptNLL, is # dsWriteBA is to do ds_write B first. # grBA is to do buffer_load B first. ############################################################################## - def loopBody( self, kernel, tensorParametersA, tensorParametersB, pack, lc, loopCopies, finalLoop, firstIter=False, dsWriteBA=False, grBA=False, isDTVGRSecondBuf=False, skipClose=False ): + def _loopBody( self, kernel, tensorParametersA, tensorParametersB, pack, lc, loopCopies, finalLoop, firstIter=False, dsWriteBA=False, grBA=False, isDTVGRSecondBuf=False, skipClose=False ): module = Module("loopBody") expand = kernel["ExpandPointerSwap"] @@ -2322,13 +2325,15 @@ def loopBody( self, kernel, tensorParametersA, tensorParametersB, pack, lc, loop ## 8x8 -> split into group of 16 MAC(s) ## supports only PLR=0 ############################################################################### - if self.states.numItersPLR or (not globalParameters["UnrollLoopEfficiencyEnable"]): - subIterCode = self.makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ - u, pointerLWCode, pointerLRCode, waitCode, macIterCode, waitLWCode, syncCode, pack[luIdx], module) - module.add(subIterCode) # add scheduled "other", local reads, local writes - pack[luIdx] = Module() - else: - printExit("TensileLite does not support MAC instructions.") + + # Is this test necessary because of the global variable this if was previously always true + # after removing the global variable it is always false... + # if self.states.numItersPLR: + subIterCode = self._makeSubIterSchedule(kernel, tensorParametersA, tensorParametersB, localReads, \ + u, pointerLWCode, pointerLRCode, waitCode, macIterCode, waitLWCode, syncCode, pack[luIdx], module) + module.add(subIterCode) # add scheduled "other", local reads, local writes + pack[luIdx] = Module() + # close unrolled loop endStr = "" if loopCopies == 2: @@ -2509,7 +2514,7 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): dsWriteBA = True if isULSGRO else False # second GR buffer check for DTV isDTVGRSecondBuf = True if isDTV else False - module.add(self.loopBody( kernel, tensorParametersA, tensorParametersB, pack, 0, loopCopies, False , dsWriteBA=dsWriteBA, isDTVGRSecondBuf=isDTVGRSecondBuf, skipClose=True)) + module.add(self._loopBody( kernel, tensorParametersA, tensorParametersB, pack, 0, loopCopies, False , dsWriteBA=dsWriteBA, isDTVGRSecondBuf=isDTVGRSecondBuf, skipClose=True)) loopLabelToNoGRloopAfterABLoop = Label("NoGRloopAfterABLoop", "" ) loopCounter = self.loopCounter(kernel, self.states.unrollIdx) module.add(SSubU32(dst=loopCounter, src0=loopCounter, \ @@ -2521,14 +2526,14 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): module.add(SCBranchSCC1(labelName=loopLabelToNoGRloopAfterABLoop.getLabelName(), comment="exit LoopL" )) # grBA check for UnrollLoopSwapGlobalReadOrder grBA = True if isULSGRO else False - module.add(self.loopBody( kernel, tensorParametersA, tensorParametersB, pack, 1, loopCopies, True , grBA=grBA)) + module.add(self._loopBody( kernel, tensorParametersA, tensorParametersB, pack, 1, loopCopies, True , grBA=grBA)) else: for lc in range(0, loopCopies): # second GR buffer check for DTV isDTVGRSecondBuf = True if isDTV and lc == 0 else False # loop body code generation finalLoop = lc == loopCopies - 1 - module.add(self.loopBody( kernel, tensorParametersA, tensorParametersB, pack, lc, loopCopies, finalLoop, isDTVGRSecondBuf=isDTVGRSecondBuf )) + module.add(self._loopBody( kernel, tensorParametersA, tensorParametersB, pack, lc, loopCopies, finalLoop, isDTVGRSecondBuf=isDTVGRSecondBuf )) module.addComment1("Before NLL: Check VGPR.checkin for INT8 LW") @@ -2942,18 +2947,19 @@ def kernelBody( self, kernel, tensorParametersA, tensorParametersB ): ############################################################################## # Init Kernel ############################################################################## - def initKernel(self, kernel, tensorParametersA, tensorParametersB): + def _initKernel(self, kernel, tensorParametersA, tensorParametersB): assert kernel["KernelLanguage"] == "Assembly" self.language = "ASM" # ISA version, such as 803 version = tuple(kernel["ISA"]) if self.ti == None: self.ti = TensileInstructions() - self.ti.init(version, self.assembler) + self.ti.init(version, str(self.assembler.path)) self.ti.setKernelInfo(version, kernel["WavefrontSize"]) + self.ti.getArchCaps self.consts = ConstValues() - self.states = StateValues(version=version, kernel=kernel, kernelName=self.getKernelName(kernel)) + self.states = StateValues(version=version, kernel=kernel, kernelName=getKernelName(self.kernelMinNaming, self.debugConfig.splitGSU, kernel)) self.vgprs = StateVgprs() self.sgprs = collections.OrderedDict() self.codes = CodeModules() @@ -3259,7 +3265,7 @@ def readWriteVectors(mat, vw, kernel): # Check if the address setup code for LWA and GRO causes register growth. # This is not an error condition but bears further investigation. # Realistically we just have the GlobalToLocal VGPRs, all else is growth. - self.states.preventVgprOverflowDuringNewTile = 0 and not globalParameters["ForceGenerateKernel"] + self.states.preventVgprOverflowDuringNewTile = 0 and not self.debugConfig.forceGenerateKernel # For Beta: # Rather than waiting for all loads to finish with s_waitcnt vmcnt(0), interleave @@ -3688,7 +3694,7 @@ def readWriteVectors(mat, vw, kernel): else: numVgprGlobalReadIncsMetadata = 0 - numVgprAddressDbg = self.states.rpga if globalParameters["DebugKernel"] else 0 + numVgprAddressDbg = self.states.rpga if self.debugConfig.debugKernel else 0 #################################### # num vgprs: c write address @@ -4040,7 +4046,7 @@ def readWriteVectors(mat, vw, kernel): self.states.numActivationTypeArgSize = 0 # Will change to 1 if activationType == All self.states.numActivationArgSize = max(1, int(kernel["ProblemType"]["DestDataType"].numRegisters())) self.states.numactivationArgTotalSize = self.states.numActivationArgSize * kernel["ProblemType"]["ActivationType"].getAdditionalArgNum() - self.states.numSgprAddressDbg = self.states.rpga if globalParameters["DebugKernel"] else 0 + self.states.numSgprAddressDbg = self.states.rpga if self.debugConfig.debugKernel else 0 #################################### # num sgprs: global read increments @@ -4112,7 +4118,7 @@ def readWriteVectors(mat, vw, kernel): self.defineSgpr("OrigLoopCounter", 1) - if globalParameters["DebugKernel"]: + if self.debugConfig.debugKernel: self.defineSgpr("AddressDbg", self.states.numSgprAddressDbg) self.defineSgpr("DebugKernelItems", 1) @@ -4386,7 +4392,7 @@ def readWriteVectors(mat, vw, kernel): # Special dependency cases if kernel["ProblemType"]["ComputeDataType"].isDouble(): if kernel["MatrixInstruction"] == [4, 4, 4, 4]: - if kernel['ISA'] == [9,0,10]: + if kernel['ISA'] == IsaVersion(9,0,10): self.states.miDependency = 4 @@ -5040,49 +5046,6 @@ def _syncThreads(self, kernel, comment="", skipForceWaitcnt0=False): # ############################################################################## - def _shortenFileBase(self, kernel): - base = self.getKernelName(kernel) - if len(base) <= MAX_FILENAME_LENGTH: - return base - - import hashlib - import base64 - - pivot = MAX_FILENAME_LENGTH * 3 // 4 - firstPart = base[:pivot] - secondPart = base[pivot:] - - secondHash = hashlib.sha256(secondPart.encode()).digest() - secondPart = base64.b64encode(secondHash, b'_-').decode() - - return firstPart + secondPart - - - def _getCustomKernelSource(self, kernel, CustomKernelDirectory): - kernelName = self.getKernelFileBase(kernel) - with open(os.path.join(CustomKernelDirectory, (kernelName + ".s"))) as f: - hipccver = globalParameters['HipClangVersion'].split(".") - hipccMaj = int(hipccver[0]) - hipccPatch = int(hipccver[2].split("-")[0]) - if not (hipccMaj >= 6 and hipccPatch >= 32650): - code = [] - for line in f.readlines(): - if "amdhsa_user_sgpr_kernarg_preload" not in line: - code.append(line) - code = "".join(code) - else: - code = f.read() - - self.tPA = tensorParametersA = {} - self.tPB = tensorParametersB = {} - self.states.kernel = kernel - self.states.language = "ASM" - self.states.version = tuple(kernel["ISA"]) if "ISA" in kernel else globalParameters["CurrentISA"] - if not globalParameters["AsmCaps"][self.states.version]["SupportedISA"]: - self.states.version = (9,0,0) - printWarning(f"ISA: {self.version} is not supported; overriding with {self.states.version}") - - return code def _getKernelSource(self, kernel: Solution): """ @@ -5092,41 +5055,24 @@ def _getKernelSource(self, kernel: Solution): fileString = "" tensorParametersA = {} tensorParametersB = {} - self.initKernel(kernel, tensorParametersA, tensorParametersB) + self._initKernel(kernel, tensorParametersA, tensorParametersB) self.stringIdx = 0 (error, kb) = self.kernelBody(kernel, tensorParametersA, tensorParametersB) fileString += str(kb) if error != 0: - if globalParameters["ForceGenerateKernel"]: + if self.debugConfig.forceGenerateKernel: printWarning("Generating kernel source resulted in error {}, but ForceGenerateKernel=1 so saving source".format(error)) else: raise RuntimeError("Generating kernel source resulted in error {}".format(error)) return fileString - ############################################################################## - # get kernel name - ############################################################################## - def getKernelFileBase(self, kernel): - if isCustomKernelConfig(kernel): - fileBase = kernel["CustomKernelName"] - elif globalParameters["ShortNames"]: - fileBase = Solution.getNameSerial(kernel, self.kernelSerialNaming) - else: - fileBase = self._shortenFileBase(kernel) - return fileBase - - def getKernelName(self, kernel): - kernelName = Solution.getNameMin(kernel, self.kernelMinNaming, True) - return kernelName - @abc.abstractmethod def getSourceFileString(self, kernel) -> Tuple[int, str]: """ Returns a string suitable for placing in Kernels.cpp. This means the actual kernel source in the case - of a source kernel, or an assembled code object byte array definition in the case of an assembly kernel, - or an empty string in the case that CodeFromFiles is true. + of a source kernel, or an assembled code object byte array definition in the case of an assembly kernel. In the case of an assembly kernel, this function has the side effect of creating the following files: * An assembly source file @@ -5137,10 +5083,9 @@ def getSourceFileString(self, kernel) -> Tuple[int, str]: pass def getHeaderFileString(self, kernel): - kernelName = self.getKernelName(kernel) + kernelName = getKernelName(self.kernelMinNaming, self.debugConfig.splitGSU, kernel) fileString = "" # CHeader - if not globalParameters["CodeFromFiles"]: - fileString += "extern const unsigned char %s_coba[]; // code object byte array\n" % kernelName + fileString += "extern const unsigned char %s_coba[]; // code object byte array\n" % kernelName return fileString diff --git a/tensilelite/Tensile/KernelWriterActivationEnumHeader.py b/tensilelite/Tensile/KernelWriterActivationEnumHeader.py index 84ba70c27b..b7b4f563f0 100644 --- a/tensilelite/Tensile/KernelWriterActivationEnumHeader.py +++ b/tensilelite/Tensile/KernelWriterActivationEnumHeader.py @@ -22,7 +22,6 @@ from copy import deepcopy -from .Common import globalParameters, CHeader from .Activation import ActivationType from .KernelWriterBase import KernelWriterBase diff --git a/tensilelite/Tensile/KernelWriterActivationFunction.py b/tensilelite/Tensile/KernelWriterActivationFunction.py index e25a4ca905..ef70b78bf1 100644 --- a/tensilelite/Tensile/KernelWriterActivationFunction.py +++ b/tensilelite/Tensile/KernelWriterActivationFunction.py @@ -21,15 +21,16 @@ ################################################################################ from copy import deepcopy +from typing import List from .TensileInstructions import TensileInstructions -from .Common import globalParameters, gfxToIsa, isaToGfx +from .Common import isaToGfx, IsaVersion from .Activation import ActivationInline, ActivationType from .KernelWriterBase import KernelWriterBase class KernelWriterActivationFunction(KernelWriterBase): - def __init__(self, state, cxxCompiler: str): + def __init__(self, state, cxxCompiler: str, supportedISA: List[IsaVersion]): super().__init__() self.cxxCompiler = cxxCompiler self.state["ProblemType"] = deepcopy(state["ProblemType"]) @@ -46,16 +47,7 @@ def __init__(self, state, cxxCompiler: str): self.enumName = "Tensile::%sActivationType_%s"%(self.actGradientPrefix, \ self.state["ProblemType"]["ActivationComputeDataType"]) - # Get supported archs - if ";" in globalParameters["Architecture"]: - self.supportedArchs = globalParameters["Architecture"].split(";") - else: - self.supportedArchs = globalParameters["Architecture"].split("_") - if "all" in self.supportedArchs: - self.supportedArchs = deepcopy(globalParameters['SupportedISA']) - else: - for idx, arch in enumerate(self.supportedArchs): - self.supportedArchs[idx] = gfxToIsa(''.join(map(str, arch))) + self.supportedArchs = supportedISA # derive parameter self.language = "HIP" @@ -93,11 +85,11 @@ def functionSignature(self): def getInlineAsm(self, activation: ActivationInline, spaces: int, activationType: str): activationStrList = [] - isa = tuple(self.state["Kernel"]["ISA"]) + isa = self.state["Kernel"]["ISA"] if not self._tf.isInit(): self._tf.init(isa, self.cxxCompiler) self._tf.setKernelInfo(isa, self.state["Kernel"]["WavefrontSize"]) - + for arch in self.supportedArchs: self._tf.init(arch, self.cxxCompiler) self._tf.setKernelInfo(arch, self.state["Kernel"]["WavefrontSize"]) diff --git a/tensilelite/Tensile/KernelWriterAssembly.py b/tensilelite/Tensile/KernelWriterAssembly.py index 6038eab8c0..ee10ba254d 100644 --- a/tensilelite/Tensile/KernelWriterAssembly.py +++ b/tensilelite/Tensile/KernelWriterAssembly.py @@ -41,20 +41,21 @@ from .TensilePass import getActivationFunctionModuleName, getActivationBranchModuleName from .TensileInstructions.Containers import HWRegContainer from .Component import Component -from .KernelWriter import KernelWriter, ConstValues, StateValues, StateVgprs, CodeModules +from .KernelWriter import KernelWriter from .KernelWriterModules import * from .SolutionStructs import isPackedIndex from .AsmStoreState import StoreState, VectorDataTypes from .AsmMemoryInstruction import MemoryInstruction from .Activation import ActivationType from .CustomKernels import isCustomKernelConfig -from .Common import globalParameters, print2, printExit, printWarning, roundUp, ensurePath, INDEX_CHARS, DataDirection, SemanticVersion -from dataclasses import dataclass +from Tensile.Common import print1, print2, printExit, printWarning, INDEX_CHARS, DebugConfig, DataDirection +from Tensile.SolutionStructs.Naming import getKernelFileBase +from Tensile.Toolchain.Component import Assembler from math import ceil, log, floor from copy import deepcopy from dataclasses import dataclass, field -from typing import NamedTuple, Tuple +from typing import NamedTuple, Tuple, Dict import os import subprocess @@ -82,10 +83,41 @@ class KernelWriterAssembly(KernelWriter): ############################################################################## # Init ############################################################################## - def __init__(self, kernelMinNaming, kernelSerialNaming, assembler: str, amdClangVersion: SemanticVersion): - super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, amdClangVersion) + def __init__( + self, + kernelMinNaming, + kernelSerialNaming, + assembler: Assembler, + debugConfig: DebugConfig, + ): + super(KernelWriterAssembly, self).__init__(kernelMinNaming, kernelSerialNaming, assembler, debugConfig) + + + def _getCustomKernelSource(self, useShortNames, kernel, CustomKernelDirectory): + kernelName = getKernelFileBase(useShortNames, self.debugConfig.splitGSU, self.kernelMinNaming, self.kernelSerialNaming, kernel) + with open(os.path.join(CustomKernelDirectory, (kernelName + ".s"))) as f: + rocmVersion = self.assembler.rocm_version + if not (rocmVersion.major >= 6 and rocmVersion.patch >= 32650): + code = [] + for line in f.readlines(): + if "amdhsa_user_sgpr_kernarg_preload" not in line: + code.append(line) + code = "".join(code) + else: + code = f.read() + + self.tPA = {} + self.tPB = {} + self.states.kernel = kernel + self.states.language = "ASM" + self.states.version = kernel["ISA"] + + return code - def getSourceFileString(self, kernel) -> Tuple[int, str]: + + def getSourceFileString(self, + kernel, + useShortNames: bool=False) -> Tuple[int, str]: assert kernel["KernelLanguage"] == "Assembly" # Skip if .o files will have already been built for this file if kernel.duplicate: @@ -93,7 +125,7 @@ def getSourceFileString(self, kernel) -> Tuple[int, str]: return (0, "") # should this be an non zero number try: - code = self._getCustomKernelSource(kernel, CUSTOM_KERNEL_PATH) if isCustomKernelConfig(kernel) else self._getKernelSource(kernel) + code = self._getCustomKernelSource(useShortNames, kernel, CUSTOM_KERNEL_PATH) if isCustomKernelConfig(kernel) else self._getKernelSource(kernel) errcode = 0 except RuntimeError as e: printWarning(f"Failed to generate assembly source code for {kernel}: {e}") @@ -780,7 +812,7 @@ def macroAndSet(self, kernel, tPA, tPB) -> Module: module.add(RegSet("v", "vgprSerial", self.states.startVgprSerial)) - if globalParameters["DebugKernel"]: + if self.debugConfig.debugKernel: module.add(RegSet("v", "vgprAddressDbg", \ self.states.startVgprAddressDbg)) #module.addComment0("Occu: %u waves/simd" % self.numWavesPerSimd ) @@ -1227,7 +1259,7 @@ def checkResources(self, kernel, mkb: KernelBody): else: msg = "unknown" - if globalParameters["PrintSolutionRejectionReason"]: + if self.debugConfig.printSolutionRejectionReason: printWarning("%s overflowed resources. errorCode=%d, msg=\"%s\", vgprs=%u, sgprs=%u" \ % (self.states.kernelName, self.states.overflowedResources, msg, \ self.vgprPool.size(), self.sgprPool.size())) @@ -1285,7 +1317,7 @@ def loadBatchedAddress(self, kernel, Batch, tmpSgprResource: RegisterPoolResourc def getKernelArgLoadModule(self, kernel, sgprStartIdx, numsOfLoad, preloadNum): kernelArgs = Module("load arguments") kernelArgs.addComment1("Load Kernel Args") - if globalParameters["DebugKernel"]: + if self.debugConfig.debugKernel: kernelArgs.add(self.argLoader.loadKernArg("AddressDbg", "KernArgAddress", dword=2)) self.argLoader.resetOffset() kernelArgs.addModuleAsFlatItems(self.argLoader.loadAllKernArg(sgprStartIdx, "KernArgAddress", numsOfLoad, preloadNum)) @@ -2021,7 +2053,7 @@ def calculateWG(): ######################################## # Debug Buffer - if globalParameters["DebugKernel"]: + if self.debugConfig.debugKernel: module.addComment1("Debug Buffer") # nwg0 FIXME use NumWorkGroups0 @@ -4369,7 +4401,7 @@ def tailLoopAllocDTVVgpr(self, kernel, tensorParametersA, tensorParametersB): numVgprValuPackA *= 2 else: numVgprValuPackA = self.states.a.numVgprValuPerBlock * kernel["InnerUnroll"] * self.states.numVgprBufferPackA * (int(4/tensorParametersA["bpeDS"]) - 1) - + vgprBaseA = self.vgprPool.checkOutAligned(numValuA + numVgprValuPackA, 2) imodA.add(RegSet("v", "vgprValuA_X0_I0_BASE", vgprBaseA)) if numVgprValuPackA > 0: @@ -4445,7 +4477,7 @@ def tailLoopAllocDTVVgpr(self, kernel, tensorParametersA, tensorParametersB): ############################################################################## # Using wider load instructions to improve the GR efficiency in tail loop. # If loading size is smaller than a dword(32bit), it will return 0 instead. - # Need to call buffer_load_d16 to load the data which is out of boundary. + # Need to call buffer_load_d16 to load the data which is out of boundary. ############################################################################## def tailLoopGlobalRead(self, kernel, tPA, tPB, doA, doB): imod = Module("tailLoopGlobalRead") @@ -4685,7 +4717,7 @@ def func(idx, bevavior, jumpLabel, tileSgpr, kSgpr): if doA and kernel["DirectToLds%s"%tPA["tensorChar"]]: imod.add(SMovB32(dst=mgpr(0), src=hex(kernel["LdsNumBytes"]), \ comment="Restore LDS clamp at %u bytes HERE"%(kernel["LdsNumBytes"]))) - + imod.add(SCmpEQU32(src0=sgpr(tmpSgprKB), src1=0, \ comment="Valid loading size per thread is multiples of 4 bytes")) @@ -5494,7 +5526,7 @@ def endSummation(self, kernel, tPA, tPB, noSkipLoad = True, label = None, isOptN # Write bias A, B data to LDS if kernel["ProblemType"]["Gradient"] and kernel["ProblemType"]["UseBias"] and (kernel["ProblemType"]["BiasSrc"] == "A" or kernel["ProblemType"]["BiasSrc"] == "B"): - + tP = tPA if kernel["ProblemType"]["BiasSrc"] == "A" else tPB module.add(self.exclasses.biasSumUnroll.storeSumLDS(self, kernel, tP)) @@ -7417,17 +7449,17 @@ def globalReadGuardKBody(tP, optParams = None): if kernel["ProblemType"]["Sparse"] and not kernel["DirectToVgprSparseMetadata"]: if tP["is_sparse"]: globalReadGuardKBody(tP["tpsMetadata"]) - + if self.db["ConservativeWaitCnt"] & 0x1: module.add(SBarrier(comment="debug")) module.add(SWaitCnt(lgkmcnt=0, vmcnt=0, vscnt=0, comment="")) module.add(SBarrier(comment="debug")) - + # TODO - can remove one of these m0 restores if A and B both TLU if kernel["DirectToLds%s"%tP["tensorChar"]]: module.add(SMovB32(dst=mgpr(0), src=hex(kernel["LdsNumBytes"]), \ comment="Restore LDS clamp at %u bytes HERE"%(kernel["LdsNumBytes"]))) - + if not kernel["BufferLoad"]: self.vgprPool.checkIn(maxAddrVgpr) self.vgprPool.checkIn(bpeVgpr) @@ -7618,7 +7650,7 @@ def globalReadDo(self, kernel, mode, tP, unrollLoopIdx=-1, g2lBufIdx=0, \ tc = tP["tensorChar"] problemType = self.states.kernel["ProblemType"] imod = StructuredModule("globalReadDo%s_%u"%(tc,mode)) - if not self.do["GlobalRead%s"%tP["tensorChar"]]: + if not self.do["GlobalRead%s"%tP["tensorChar"]]: return imod # sizeK % LOCAL_DEPTHU @@ -7781,7 +7813,7 @@ def globalReadBody(tP): else: g2lIdxM = i * max(loadWidth * tP["bpeRatio"], 1) destVgpr = destVgprPrefix + "+%u"%((g2lIdx+eccOffset+tP["shiftGR"]) if not tP["isM"] else g2lIdxM) - self.vgprs.globalReadRegisters[tc].append(g2lIdx+eccOffset+tP["shiftGR"] if not tP["isM"] else g2lIdxM) + self.vgprs.globalReadRegisters[tc].append(g2lIdx+eccOffset+tP["shiftGR"] if not tP["isM"] else g2lIdxM) if tP["isM"]: assert(graIdx <= self.states.m.numVgprG2LAllocated) @@ -9979,7 +10011,7 @@ def globalWriteElements(self, kernel, tPA, tPB, vectorWidths_2, vectorWidths_1, useBiasBackup = self.states.useBias betasBackup = betas edgesBackup = edges - gsuLimit = 1 if noGSUBranch or globalParameters["SplitGSU"] else 2 + gsuLimit = 1 if noGSUBranch or self.debugConfig.splitGSU else 2 if gsuLimit > 1: gsuLabel = Label(label=self.labels.getNameInc("GSU"), comment="") with self.allocTmpSgpr(1) as tmpSgprGSU: @@ -11491,7 +11523,7 @@ def globalWriteBatch(self, kernel, tPA, tPB, activation, ss: StoreState, batchId addrScaleAVec, addrScaleBVec, addrScaleAlphaVec, biasLocalBarrierInit, \ tmpVgpr, tmpVgprDynamic, cvtVgprStruct, activationSetPCStruct, activationTypeStr, \ batchElementSgprs, tmpSgpr, codeAccVgprRead, codeMulAlpha, packdata, self, factorDim, \ - self.amdClangVersion) + self.assembler.version) ############################################################################## def openPrefetchGlobalRead2(self, kernel): diff --git a/tensilelite/Tensile/KernelWriterBetaOnly.py b/tensilelite/Tensile/KernelWriterBetaOnly.py index c3f750c361..b49a856a0e 100644 --- a/tensilelite/Tensile/KernelWriterBetaOnly.py +++ b/tensilelite/Tensile/KernelWriterBetaOnly.py @@ -24,7 +24,7 @@ from copy import deepcopy -from .Common import globalParameters, CHeader, INDEX_CHARS +from .Common import INDEX_CHARS from .TensileInstructions import DataType from .KernelWriterBase import KernelWriterBase diff --git a/tensilelite/Tensile/KernelWriterConversion.py b/tensilelite/Tensile/KernelWriterConversion.py index 582205c7cf..c53959b915 100644 --- a/tensilelite/Tensile/KernelWriterConversion.py +++ b/tensilelite/Tensile/KernelWriterConversion.py @@ -22,16 +22,18 @@ # ################################################################################ +from typing import Dict from copy import deepcopy +from typing import List from .KernelWriterBase import KernelWriterBase from .TensileInstructions import DataType -from .Common import globalParameters, gfxToIsa, isaToGfx, INDEX_CHARS +from .Common import IsaInfo, isaToGfx, INDEX_CHARS class KernelWriterConversion(KernelWriterBase): - def __init__(self, state, load_vw): + def __init__(self, state, load_vw, supportedArchs: List[tuple], isaInfoMap: Dict[str, IsaInfo]): super().__init__() self.state["ProblemType"] = deepcopy(state["ProblemType"]) @@ -65,6 +67,7 @@ def __init__(self, state, load_vw): # derive parameter self.language = "HIP" self.kernelName = self.getKernelName() + self.isaInfoMap = isaInfoMap self.datatype = self.state["ProblemType"]["ComputeDataType"].toDevice(self.language) self.int32Str = DataType('int32').toDevice(self.language) if self.state["ProblemType"]["DataType"].isInt8() and self.state["ProblemType"]["ComputeDataType"].isSingle() and self.state["ProblemType"]["HighPrecisionAccumulate"]: @@ -80,15 +83,7 @@ def __init__(self, state, load_vw): self.tileChar1 = self.indexChars[self.state["ProblemType"]["Index1"]] # Get supported archs - if ";" in globalParameters["Architecture"]: - self.supportedArchs = globalParameters["Architecture"].split(";") - else: - self.supportedArchs = globalParameters["Architecture"].split("_") - if "all" in self.supportedArchs: - self.supportedArchs = deepcopy(globalParameters['SupportedISA']) - else: - for idx, arch in enumerate(self.supportedArchs): - self.supportedArchs[idx] = gfxToIsa(''.join(map(str, arch))) + self.supportedArchs = supportedArchs self.gsuKernels = [self.state["GlobalSplitU"]] if self.state["GenPGRPostKernels"]: @@ -532,9 +527,9 @@ def kernelBody(self): if self.num_dword_load > 2: kStr += " float2 accumVec2(accum[2], accum[3]);" + self.endLine canPKF32Arch = [] - for arch in self.supportedArchs: - archTuple = tuple(arch) - if globalParameters["AsmCaps"][archTuple]['v_pk_add_f32']: + for arch in self.supportedArchs: # certainly we can move this out to the __init__ + isa = tuple(arch) + if self.isaInfoMap[isa].asmCaps['v_pk_add_f32']: canPKF32Arch.append(arch) defineStr = [] if len(canPKF32Arch) > 0: diff --git a/tensilelite/Tensile/LibraryIO.py b/tensilelite/Tensile/LibraryIO.py index a266a5b4b9..256d530b4d 100644 --- a/tensilelite/Tensile/LibraryIO.py +++ b/tensilelite/Tensile/LibraryIO.py @@ -23,12 +23,15 @@ ################################################################################ from .CustomKernels import getCustomKernelConfig -from .SolutionStructs import Solution, ProblemSizes, ProblemType +from .SolutionStructs import Solution, ProblemSizes +from Tensile.SolutionStructs.Problem import ProblemType from . import SolutionLibrary from .CustomYamlLoader import load_yaml_stream -from .Common import gfxToIsa, printExit, printWarning, print2, versionIsCompatible, __version__ +from .Common import gfxToIsa, printExit, printWarning, print2, \ + versionIsCompatible, __version__, \ + IsaInfo, DepthUConfig -from typing import NamedTuple, List +from typing import NamedTuple, List, Dict import os import sys @@ -148,6 +151,8 @@ def writeSolutions(filename, problemSizes, biasTypeArgs, activationArgs, solutio if "DataTypeMetadata" in solutionState["ProblemType"]: solutionState["ProblemType"]["DataTypeMetadata"] = \ solutionState["ProblemType"]["DataTypeMetadata"].value + isa = solutionState["ISA"] + solutionState["ISA"] = [isa[0], isa[1], isa[2]] solutionStates.append(solutionState) # write dictionaries with open(filename, "w") as f: @@ -180,24 +185,53 @@ def read(filename, customizedLoader=False): else: printExit("Unrecognized read format {}".format(extension)) + def readYAML(filename): """Reads and returns YAML data from file.""" with open(filename, "r") as f: data = yaml.load(f, yamlLoader) return data + def readJson(filename): """Reads and returns JSON data from file.""" with open(filename, "r") as f: data = json.loads(f.read()) return data -def parseSolutionsFile(filename, cxxCompiler): - """Wrapper function to read and parse a solutions file.""" - return parseSolutionsData(read(filename), filename, cxxCompiler) - -def parseSolutionsData(data, srcFile, cxxCompiler): +def parseSolutionsFile( + filename, + assembler, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + isaInfoMap + ): + """Wrapper function to read and parse a solutions file.""" + return parseSolutionsData( + read(filename), + filename, + assembler, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + isaInfoMap + ) + + +def parseSolutionsData( + data, + srcFile, + assembler, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + isaInfoMap + ): """Parses problem sizes and solutions from the data of a solutions file.""" if len(data) < 3: printExit("Solution file {} is missing required fields (len = {} < 3" \ @@ -224,7 +258,16 @@ def parseSolutionsData(data, srcFile, cxxCompiler): # force redo the deriving of parameters, make sure old version logic yamls can be validated solutionState["AssignedProblemIndependentDerivedParameters"] = False solutionState["AssignedDerivedParameters"] = False - solutionObject = Solution(solutionState, cxxCompiler, srcFile) + solutionObject = Solution( + solutionState, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap, + srcFile + ) solutions.append(solutionObject) problemType = solutions[0]["ProblemType"] problemSizes = ProblemSizes(problemType, problemSizesConfig) @@ -240,25 +283,54 @@ class LibraryLogic(NamedTuple): exactLogic: list library: SolutionLibrary.MasterSolutionLibrary -def parseLibraryLogicFile(filename, cxxCompiler, archs=None): +def parseLibraryLogicFile( + filename, + assembler, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + isaInfoMap: Dict[str, IsaInfo], + lazyLibraryLoading: bool + ): """Wrapper function to read and parse a library logic file.""" - return parseLibraryLogicData(read(filename, True), filename, cxxCompiler, archs) - - -def parseLibraryLogicData(data, srcFile, cxxCompiler, archs=None): + return parseLibraryLogicData( + read(filename, True), + filename, + assembler, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + isaInfoMap, + lazyLibraryLoading + ) + + +def parseLibraryLogicData( + data, + srcFile, + assembler, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + isaInfoMap: Dict[str, IsaInfo], + lazyLibraryLoading: bool + ): """Parses the data of a library logic file.""" if isinstance(data, List): data = parseLibraryLogicList(data, srcFile) - is_arch_valid = lambda cArch, tArch : (cArch == tArch or cArch == "all") - if not (archs is None) and "ArchitectureName" in data: - if isinstance(archs, List): - if len(archs) > 0 and not archs[0] == "all": - if not (any(is_arch_valid(arch.split(":")[0], data["ArchitectureName"]) for arch in archs)): - return LibraryLogic("", "", None, [], [], None, srcFile) - elif isinstance(archs, str): - if not is_arch_valid(archs.split(":")[0], data["ArchitectureName"]): - return LibraryLogic("", "", None, [], [], None, srcFile) + #is_arch_valid = lambda cArch, tArch : (cArch == tArch or cArch == "all") + #if not (archs is None) and "ArchitectureName" in data: + # if isinstance(archs, List): + # if len(archs) > 0 and not archs[0] == "all": + # if not (any(is_arch_valid(arch.split(":")[0], data["ArchitectureName"]) for arch in archs)): + # return LibraryLogic("", "", None, [], [], None, srcFile) + # elif isinstance(archs, str): + # if not is_arch_valid(archs.split(":")[0], data["ArchitectureName"]): + # return LibraryLogic("", "", None, [], [], None, srcFile) if "CUCount" not in data: data["CUCount"] = None @@ -268,14 +340,12 @@ def parseLibraryLogicData(data, srcFile, cxxCompiler, archs=None): .format(srcFile, data["MinimumRequiredVersion"], __version__) ) # unpack problemType - problemType = ProblemType(data["ProblemType"]) + problemType = ProblemType(data["ProblemType"], printIndexAssignmentInfo) # unpack solution - def solutionStateToSolution(solutionState, cxxCompiler) -> Solution: + def solutionStateToSolution(solutionState, assembler, isaInfoMap) -> Solution: if solutionState["KernelLanguage"] == "Assembly": solutionState["ISA"] = gfxToIsa(data["ArchitectureName"]) - else: - solutionState["ISA"] = (0, 0, 0) solutionState["CUCount"] = data["CUCount"] # force redo the deriving of parameters, make sure old version logic yamls can be validated solutionState["AssignedProblemIndependentDerivedParameters"] = False @@ -287,10 +357,24 @@ def solutionStateToSolution(solutionState, cxxCompiler) -> Solution: customConfig = getCustomKernelConfig(solutionState["CustomKernelName"], isp) for key, value in customConfig.items(): solutionState[key] = value + + if len(customConfig["MatrixInstruction"]) != 4: + raise ValueError(f"Custom kernel MatrixInstruction can only be of length 4, found {customConfig['MatrixInstruction']}") + # The ActivationType setting in YAML is meaningless in customKernel case. # Therefore, we override the customKernel setting with the ActivationType value from ProblemType to avoid false alarms during subsequent problemType checks. solutionState["ProblemType"]["ActivationType"] = problemType["ActivationType"] - solutionObject = Solution(solutionState, cxxCompiler, srcFile) + + solutionObject = Solution( + solutionState, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap, + srcFile + ) solutionProblemType = solutionObject["ProblemType"] if problemType != solutionProblemType: # find the mismatched items in ProblemType @@ -302,9 +386,19 @@ def solutionStateToSolution(solutionState, cxxCompiler) -> Solution: printExit(f"ProblemType in library logic file {srcFile} doesn't match solution(idx={solIdx}): \n{results}") return solutionObject - solutions = [solutionStateToSolution(solutionState, cxxCompiler) for solutionState in data["Solutions"]] - - newLibrary, _ = SolutionLibrary.MasterSolutionLibrary.FromOriginalState(data, solutions, cxxCompiler) + solutions = [solutionStateToSolution(solutionState, assembler, isaInfoMap) for solutionState in data["Solutions"]] + + newLibrary, _ = SolutionLibrary.MasterSolutionLibrary.FromOriginalState( + data, + solutions, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap, + lazyLibraryLoading + ) return LibraryLogic(data["ScheduleName"], data["ArchitectureName"], problemType, solutions, \ data.get("ExactLogic"), newLibrary) @@ -464,6 +558,8 @@ def createLibraryLogic(schedulePrefix, architectureName, deviceNames, libraryTyp if "DataTypeMetadata" in solutionState["ProblemType"]: solutionState["ProblemType"]["DataTypeMetadata"] = \ solutionState["ProblemType"]["DataTypeMetadata"].value + isa = solutionState["ISA"] + solutionState["ISA"] = [isa[0], isa[1], isa[2]] solutionList.append(solutionState) if tileSelection: diff --git a/tensilelite/Tensile/LibraryLogic.py b/tensilelite/Tensile/LibraryLogic.py index c8de72ff2a..fa6fe6298c 100644 --- a/tensilelite/Tensile/LibraryLogic.py +++ b/tensilelite/Tensile/LibraryLogic.py @@ -23,10 +23,11 @@ ################################################################################ from pathlib import Path +from typing import Dict from .Common import print1, print2, HR, printExit, defaultAnalysisParameters, globalParameters, \ assignParameterWithDefault, startTime, ProgressBar, printWarning, ensurePath, \ - LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR -from .SolutionStructs import Solution + LIBRARY_LOGIC_DIR, BENCHMARK_DATA_DIR, verbosity, IsaInfo, DepthUConfig +from Tensile.SolutionStructs.Naming import getMinNaming, getNameMin, getNameFull from . import LibraryIO from . import SolutionSelectionLibrary @@ -42,7 +43,7 @@ ################################################################################ # Analyze Problem Type ################################################################################ -def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryLogicPath): +def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryLogicPath, splitGSU: bool): print2(HR) print1("# Analyzing: %s" % problemType) @@ -70,21 +71,21 @@ def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryL solutions = problemSizeGroup[4] problemSizesList.append(problemSizes) solutionsList.append(solutions) - solutionMinNaming = Solution.getMinNaming(solutions) + solutionMinNaming = getMinNaming(solutions) print1("# Read: %s" % (solutionsFileName)) print2("# ProblemSizes: %s" % problemSizes) print2("# Solutions:") solutionIdx = 0 for solution in solutions: - print2("# (%u) %s" % (solutionIdx, Solution.getNameMin(solution, \ - solutionMinNaming))) + print2("# (%u) %s" % (solutionIdx, getNameMin(solution, \ + solutionMinNaming, splitGSU))) solutionIdx += 1 print2(HR) ###################################### # Create Logic Analyzer logicAnalyzer = LogicAnalyzer( problemType, problemSizesList, solutionsList, \ - dataFileNameList, inputParameters) + dataFileNameList, inputParameters, splitGSU) selectionSolutionsIdsList = None selectionSolutions = None @@ -107,7 +108,7 @@ def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryL printExit("Bad KeepLogic=%u"%globalParameters["KeepLogic"]) # print raw data - if globalParameters["PrintLevel"] >= 2: + if verbosity >= 2: line = "After Removals:\n" numOther = 1 for size in logicAnalyzer.numProblemSizes: @@ -126,9 +127,9 @@ def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryL for i in range(0, len(logicAnalyzer.solutions)): s = logicAnalyzer.solutions[i] s["SolutionIndex"] = i - s["SolutionNameMin"] = Solution.getNameMin(s, solutionMinNaming) - s["KernelNameMin"] = Solution.getNameMin(s, solutionMinNaming, True) - print1("(%2u) %s : %s" % (i, Solution.getNameMin(s, solutionMinNaming), Solution.getNameFull(s))) + s["SolutionNameMin"] = getNameMin(s, solutionMinNaming, splitGSU) + s["KernelNameMin"] = getNameMin(s, solutionMinNaming, splitGSU, True) + print1("(%2u) %s : %s" % (i, getNameMin(s, solutionMinNaming, splitGSU), getNameFull(s, splitGSU))) if enableTileSelection: validSelectionSolutions = SolutionSelectionLibrary.analyzeSolutionSelection(problemType, selectionFileNameList, \ @@ -160,8 +161,8 @@ def analyzeProblemType(problemType, problemSizeGroups, inputParameters, libraryL (validSolution, validSolutionInfo) = validSelectionSolution selectionSolutionIndex = solutionsStartIndex + i selectionSolutionsIds.add(selectionSolutionIndex) - validSolution["SolutionNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming) - validSolution["KernelNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming, True) + validSolution["SolutionNameMin"] = getNameMin(validSolution, solutionMinNaming, splitGSU) + validSolution["KernelNameMin"] = getNameMin(validSolution, solutionMinNaming, splitGSU, True) validSolution["Ideals"] = validSolutionInfo selectionSolutions.append(validSolution) @@ -250,10 +251,11 @@ class LogicAnalyzer: # ENTRY: Init ############################################################################## def __init__(self, problemType, problemSizesList, solutionsList, \ - dataFileNameList, inputParameters): + dataFileNameList, inputParameters, splitGSU: bool): # parameters self.parameters = inputParameters + self.splitGSU = splitGSU # problem type self.problemType = problemType @@ -290,12 +292,12 @@ def __init__(self, problemType, problemSizesList, solutionsList, \ self.solutionGroupMap[solutionGroupIdx][solutionIdx] = sIdx progressBar.increment() self.numSolutions = len(self.solutions) - self.solutionMinNaming = Solution.getMinNaming(self.solutions) + self.solutionMinNaming = getMinNaming(self.solutions) self.solutionNames = [] self.solutionTiles = [] for solution in self.solutions: - self.solutionNames.append(Solution.getNameMin(solution, \ - self.solutionMinNaming)) + self.solutionNames.append(getNameMin(solution, \ + self.solutionMinNaming, self.splitGSU)) self.solutionTiles.append("%ux%u"%(solution["MacroTile0"], \ solution["MacroTile1"])) self.flopsPerMac = self.problemType["DataType"].flopsPerMac() @@ -688,7 +690,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): currentIndexRange[self.indexOrder[2]][0], \ currentIndexRange[self.indexOrder[3]][0]) tab = self.tab[cii] - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write("\n%s"%tab) currentIndex = self.indexOrder[currentIndexIndex] print2("%senRule(%s)" % (tab, currentIndexRange)) @@ -714,7 +716,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): print2("%sSingleProblem & LastIndex :: winnerIdx<0; returning" % (tab) ) return None ruleList.append([-1, winnerIdx]) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write("%") ######################################## @@ -730,7 +732,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): return None rule = [ -1, nextRule ] ruleList.append(rule) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write("%") else: @@ -783,7 +785,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): initialRule = [ currentIndexRange[currentIndex][0], nextRule ] ruleList.append(initialRule) print2("%sMultiProblem::InitialRuleList=%s" % (tab, ruleList)) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write("#") ######################################## @@ -808,7 +810,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): if winnerIdx < 0: ruleList[len(ruleList)-1][0] = problemIndex # NO_UPDATE print2("%sUpdating range b/c None" % tab) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write(" ") continue else: @@ -821,7 +823,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): if nextRule == None: ruleList[len(ruleList)-1][0] = problemIndex # NO_UPDATE print2("%sUpdating b/c None" % tab) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write(" ") continue else: @@ -832,7 +834,7 @@ def enRule(self, currentIndexIndex, currentIndexRange): if candidateRule[1] == priorRule[1]: print2("%sCandidateRule==PriorRule; just updating prior" % (tab)) ruleList[len(ruleList)-1][0] = problemIndex # NO_UPDATE - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write(" ") continue @@ -872,14 +874,14 @@ def enRule(self, currentIndexIndex, currentIndexRange): if True: # or candidateRuleScore < priorRuleScore: ruleList.append(candidateRule) print2("%sAppending b/c Different" % tab) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write("#") ######################################## # prior wins else: print2("%sPrior Rule Wins" % tab) - if globalParameters["PrintLevel"] == 1: + if verbosity == 1: stdout.write(".") ruleList[len(ruleList)-1][0] = problemIndex # NO_UPDATE @@ -1117,12 +1119,12 @@ def removeSolution(self, removeSolutionIdx): for i in range(0, oldNumSolutions): if i != removeSolutionIdx: self.solutions.append(oldSolutions[i]) - self.solutionMinNaming = Solution.getMinNaming(self.solutions) + self.solutionMinNaming = getMinNaming(self.solutions) self.solutionNames = [] self.solutionTiles = [] for solution in self.solutions: - self.solutionNames.append(Solution.getNameMin(solution, \ - self.solutionMinNaming)) + self.solutionNames.append(getNameMin(solution, \ + self.solutionMinNaming, self.splitGSU)) self.solutionTiles.append("%ux%u"%(solution["MacroTile0"], \ solution["MacroTile1"])) self.numSolutions = len(self.solutions) @@ -1168,12 +1170,12 @@ def pruneSolutions(self, keepSolutions): else: removeSolutionIdxList.append(i) - self.solutionMinNaming = Solution.getMinNaming(self.solutions) + self.solutionMinNaming = getMinNaming(self.solutions) self.solutionNames = [] self.solutionTiles = [] for solution in self.solutions: - self.solutionNames.append(Solution.getNameMin(solution, \ - self.solutionMinNaming)) + self.solutionNames.append(getNameMin(solution, \ + self.solutionMinNaming, self.splitGSU)) self.solutionTiles.append("%ux%u"%(solution["MacroTile0"], \ solution["MacroTile1"])) self.numSolutions = len(self.solutions) @@ -1429,8 +1431,17 @@ def indicesToSerial(self, solutionIdx, indices ): return serial - -def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str): +def generateLogic( + config, + benchmarkDataPath, + libraryLogicPath, + cxxCompiler: str, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + isaInfoMap: Dict[str, IsaInfo] + ): libraryLogicPath = ensurePath(libraryLogicPath) @@ -1473,7 +1484,15 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str) printExit("%s doesn't exist for %s" % (dataFileName, fileBase) ) if not os.path.exists(solutionsFileName): printExit("%s doesn't exist for %s" % (solutionsFileName, fileBase) ) - (problemSizes, solutions) = LibraryIO.parseSolutionsFile(solutionsFileName, cxxCompiler) + (problemSizes, solutions) = LibraryIO.parseSolutionsFile( + solutionsFileName, + cxxCompiler, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + isaInfoMap + ) if len(solutions) == 0: printExit("%s doesn't contains any solutions." % (solutionsFileName) ) problemType = solutions[0]["ProblemType"] @@ -1483,7 +1502,7 @@ def generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler: str) dataFileName, solutionsFileName, selectionFileName, solutions) ) for problemType in problemTypes: - logicTuple = analyzeProblemType(problemType, problemTypes[problemType], analysisParameters, libraryLogicPath) + logicTuple = analyzeProblemType(problemType, problemTypes[problemType], analysisParameters, libraryLogicPath, splitGSU) filename = os.path.join(libraryLogicPath, \ "{}_{}".format(analysisParameters["ScheduleName"], str(problemType))) @@ -1546,7 +1565,26 @@ def read_max_freq(): ### ################################################################################ ################################################################################ -def main(config, cxxCompiler: str, outputPath: Path): +def main( + config, + cxxCompiler: str, + outputPath: Path, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + isaInfoMap: Dict[str, IsaInfo] + ): benchmarkDataPath = outputPath / BENCHMARK_DATA_DIR libraryLogicPath = outputPath / LIBRARY_LOGIC_DIR - generateLogic(config, benchmarkDataPath, libraryLogicPath, cxxCompiler) + generateLogic( + config, + benchmarkDataPath, + libraryLogicPath, + cxxCompiler, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + isaInfoMap + ) diff --git a/tensilelite/Tensile/Ops/AMaxGenerator.py b/tensilelite/Tensile/Ops/AMaxGenerator.py index 79682e8c4f..ded3272fa0 100644 --- a/tensilelite/Tensile/Ops/AMaxGenerator.py +++ b/tensilelite/Tensile/Ops/AMaxGenerator.py @@ -33,7 +33,7 @@ from contextlib import contextmanager import Tensile.TensileInstructions as ti from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ - assignGlobalParameters, isaToGfx, gfxToIsa, globalParameters + assignGlobalParameters, isaToGfx, gfxToIsa from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain def kernel_header(name: str, gfx_arch: str, vgpr: int, sgpr: int, lds: int): @@ -848,8 +848,7 @@ def meta_str(kernels: Tuple[KernelMeta]): if any([not i for i in (arch, toolchain_path, isa)]): restoreDefaultGlobalParameters() assignGlobalParameters({}) - detectGlobalCurrentISA() - isa = globalParameters['CurrentISA'] + isa = detectGlobalCurrentISA(0) arch = isaToGfx(isa) toolchain_path = validateToolchain(ToolchainDefaults.CXX_COMPILER) diff --git a/tensilelite/Tensile/Ops/LayerNormGenerator.py b/tensilelite/Tensile/Ops/LayerNormGenerator.py index 2df20f2d17..d0af0dbf5c 100644 --- a/tensilelite/Tensile/Ops/LayerNormGenerator.py +++ b/tensilelite/Tensile/Ops/LayerNormGenerator.py @@ -33,7 +33,7 @@ from contextlib import contextmanager import Tensile.TensileInstructions as ti from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ - assignGlobalParameters, isaToGfx, gfxToIsa, globalParameters + assignGlobalParameters, isaToGfx, gfxToIsa from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain def kernel_header(name: str, gfx_arch: str, vgpr: int, sgpr: int, lds: int): @@ -925,8 +925,7 @@ def meta_str(kernels: Tuple[KernelMeta]): if any([not i for i in (arch, toolchain_path, isa)]): restoreDefaultGlobalParameters() assignGlobalParameters({}) - detectGlobalCurrentISA() - isa = globalParameters['CurrentISA'] + isa = detectGlobalCurrentISA(0) arch = isaToGfx(isa) toolchain_path = validateToolchain(ToolchainDefaults.CXX_COMPILER) diff --git a/tensilelite/Tensile/Ops/SoftmaxGenerator.py b/tensilelite/Tensile/Ops/SoftmaxGenerator.py index 34ec8c0cd2..5471dc1d08 100644 --- a/tensilelite/Tensile/Ops/SoftmaxGenerator.py +++ b/tensilelite/Tensile/Ops/SoftmaxGenerator.py @@ -32,7 +32,7 @@ from contextlib import contextmanager import Tensile.TensileInstructions as ti from Tensile.Common import detectGlobalCurrentISA, restoreDefaultGlobalParameters, \ - assignGlobalParameters, isaToGfx, gfxToIsa, globalParameters + assignGlobalParameters, isaToGfx, gfxToIsa from Tensile.Toolchain.Validators import ToolchainDefaults, validateToolchain def record_num_calls(f): @@ -692,8 +692,7 @@ def meta_str(kernels: Tuple[KernelMeta]): if any([not i for i in (arch, toolchain_path, isa)]): restoreDefaultGlobalParameters() assignGlobalParameters({}) - detectGlobalCurrentISA() - isa = globalParameters['CurrentISA'] + isa = detectGlobalCurrentISA(0) arch = isaToGfx(isa) toolchain_path = validateToolchain(ToolchainDefaults.CXX_COMPILER) diff --git a/tensilelite/Tensile/SolutionLibrary.py b/tensilelite/Tensile/SolutionLibrary.py index 8fedd3396e..258baae39a 100644 --- a/tensilelite/Tensile/SolutionLibrary.py +++ b/tensilelite/Tensile/SolutionLibrary.py @@ -23,13 +23,14 @@ ################################################################################ import itertools +from typing import Dict from . import Properties from . import Hardware -from . import Common from . import Contractions from .SolutionStructs import Solution as OriginalSolution -from .Common import state +from .Common import state, IsaInfo, gfxToIsa, DepthUConfig +from Tensile.SolutionStructs.Naming import getMinNaming, getNameMin class SingleSolutionLibrary: Tag = "Single" @@ -300,7 +301,13 @@ def FixSolutionIndices(cls, solutions): def FromOriginalState(cls, origData, origSolutions, - cxxCompiler, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + assembler, + isaInfoMap: Dict[str, IsaInfo], + lazyLibraryLoading: bool, solutionClass=Contractions.Solution, libraryOrder=None, placeholderName='TensileLibrary'): @@ -314,7 +321,7 @@ def hardware(d, problemType, solutions, library, placeholderName): if devicePart == "fallback": pred = Hardware.HardwarePredicate("TruePred") else: - pred = Hardware.HardwarePredicate.FromHardware(Common.gfxToIsa(devicePart), cuCount) + pred = Hardware.HardwarePredicate.FromHardware(gfxToIsa(devicePart), cuCount) newLib.rows.append({"predicate": pred, "library": library}) @@ -397,7 +404,7 @@ def selection(d, problemType, solutions, library, placeholderName): else: assert 0 and "Unrecognized LibraryType." - if Common.globalParameters["LazyLibraryLoading"]: + if lazyLibraryLoading: placeholderName += '_' + str(problemType.aType) + str(problemType.bType) placeholderName += '_' + str(problemType.cType) + str(problemType.computeInputType) if problemType.activationType != 'none': @@ -442,7 +449,7 @@ def selection(d, problemType, solutions, library, placeholderName): # end library creation functions if libraryOrder is None: - if Common.globalParameters["LazyLibraryLoading"]: + if lazyLibraryLoading: libraryOrder = [ hardware, operationIdentifier, performanceMetric, predicates, placeholder, selection @@ -460,7 +467,13 @@ def selection(d, problemType, solutions, library, placeholderName): lazyLibrary, placeholderName = \ MasterSolutionLibrary.FromOriginalState(origData, origSolutions, - cxxCompiler, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap, + lazyLibraryLoading, solutionClass, libraryOrder[placeholderIndex:], placeholderName) @@ -468,7 +481,15 @@ def selection(d, problemType, solutions, library, placeholderName): origSolutions = [] problemType = Contractions.ProblemType.FromOriginalState(origData["ProblemType"]) - allSolutions = [solutionClass.FromSolutionStruct(s, cxxCompiler) for s in origSolutions] + allSolutions = [solutionClass.FromSolutionStruct( + s, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap + ) for s in origSolutions] cls.FixSolutionIndices(allSolutions) # library is constructed in reverse order i.e. bottom-up @@ -489,8 +510,25 @@ def selection(d, problemType, solutions, library, placeholderName): return rv, placeholderName @classmethod - def BenchmarkingLibrary(cls, solutions, cxxCompiler): - solutionObjs = list([Contractions.Solution.FromOriginalState(s._state, cxxCompiler) for s in solutions]) + def BenchmarkingLibrary( + cls, + solutions, + assembler, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + isaInfoMap + ): + solutionObjs = list([Contractions.Solution.FromOriginalState( + s._state, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + depthUConfig, + assembler, + isaInfoMap) + for s in solutions]) cls.FixSolutionIndices(solutionObjs) predRows = list([{ @@ -519,14 +557,14 @@ def state(self): rv["version"] = self.version return rv - def applyNaming(self, naming=None): + def applyNaming(self, splitGSU: bool, naming=None): if naming is None: kernels = itertools.chain(s.originalSolution.getKernels() for s in self.solutions.values()) - naming = OriginalSolution.getMinNaming(kernels) + naming = getMinNaming(kernels) for s in list(self.solutions.values()): - s.name = OriginalSolution.getNameMin(s.originalSolution.getKernels()[0], naming) - s.kernelName = OriginalSolution.getNameMin(s.originalSolution.getKernels()[0], naming, True) + s.name = getNameMin(s.originalSolution.getKernels()[0], naming, splitGSU) + s.kernelName = getNameMin(s.originalSolution.getKernels()[0], naming, splitGSU, True) def remapSolutionIndicesStartingFrom(self, curIndex): reIndexMap = {} diff --git a/tensilelite/Tensile/SolutionSelectionLibrary.py b/tensilelite/Tensile/SolutionSelectionLibrary.py index bafe0c0755..df7dd17bdd 100644 --- a/tensilelite/Tensile/SolutionSelectionLibrary.py +++ b/tensilelite/Tensile/SolutionSelectionLibrary.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ # ################################################################################ -from .SolutionStructs import Solution +from Tensile.SolutionStructs.Naming import getNameMin import csv @@ -89,8 +89,9 @@ def updateValidSolutions(validSolutions, analyzerSolutions, solutionMinNaming): (validSolution, validSolutionInfo) = validSelectionSolution selectionSolutionIndex = solutionsStartIndex + i selectionSolutionsIds.add(selectionSolutionIndex) - validSolution["SolutionNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming) - validSolution["KernelNameMin"] = Solution.getNameMin(validSolution, solutionMinNaming, True) + splitGSU = False # this is a reminder that we need to add this in to the function signature + validSolution["SolutionNameMin"] = getNameMin(validSolution, solutionMinNaming, splitGSU) + validSolution["KernelNameMin"] = getNameMin(validSolution, solutionMinNaming, splitGSU, True) validSolution["Ideals"] = validSolutionInfo selectionSolutions.append(validSolution) diff --git a/tensilelite/Tensile/SolutionStructs/Naming.py b/tensilelite/Tensile/SolutionStructs/Naming.py new file mode 100644 index 0000000000..41c15c0828 --- /dev/null +++ b/tensilelite/Tensile/SolutionStructs/Naming.py @@ -0,0 +1,284 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ +from copy import deepcopy +from functools import lru_cache +from typing import List + +from Tensile.Common.Constants import MAX_FILENAME_LENGTH +from Tensile.Common.ValidParameters import validParameters + +from .Problem import ProblemType + +######################################## +# create a dictionary with booleans on whether to include parameter in name +def getMinNaming(objs: list): + nonCKObjs = [obj for obj in objs if not ("CustomKernelName" in obj and obj["CustomKernelName"])] + # early return + if len(nonCKObjs) == 0: + return {} + # determine keys + requiredParameters = {} + if hasattr(nonCKObjs[0], "_state"): + keys = list(nonCKObjs[0]._state.keys()) + else: + keys = list(nonCKObjs[0].keys()) + # only 1, rather than name being nothing, it'll be everything + if len(nonCKObjs) == 1: + for key in keys: + if key in list(validParameters.keys()): + requiredParameters[key] = False + else: + for key in keys: + required = False + if key in list(validParameters.keys()): + for i in range(1, len(nonCKObjs)): + if nonCKObjs[0][key] != nonCKObjs[i][key]: + required = True + break + if required: + requiredParameters[key] = True + else: + requiredParameters[key] = False + requiredParameters["GlobalSplitU"] = True + requiredParameters["WorkGroupMapping"] = True + if "MatrixInstM" in nonCKObjs[0]._state: + # Use MIWaveGroup and MIWaveTile instead of WG and MT + requiredParameters["MIWaveTile"] = True + requiredParameters["ThreadTile"] = False + requiredParameters["ProblemType"] = False # always prepended + requiredParameters["MacroTile0"] = False # always prepended + requiredParameters["MacroTile1"] = False # always prepended + requiredParameters["DepthU"] = False # always prepended + requiredParameters["MatrixInstruction"] = False # always prepended + requiredParameters["MatrixInstM"] = False # always prepended + requiredParameters["MatrixInstN"] = False # always prepended + requiredParameters["MatrixInstK"] = False # always prepended + requiredParameters["MatrixInstB"] = False # always prepended + requiredParameters["MatrixInstBM"] = False # always prepended + requiredParameters["MatrixInstBN"] = False # always prepended + requiredParameters["CustomKernelName"] = False # Will not affect naming + requiredParameters["Kernel"] = True # distinguish kernels from solutions + # for single-source compilation + return requiredParameters + + +def getKeyNoInternalArgs(state, splitGSU: bool): + state_copy = deepcopy(state) + state_copy["ProblemType"]["GroupedGemm"] = False + if splitGSU: + state_copy["GlobalSplitU"] = "M" if (state_copy["GlobalSplitU"] > 1) else state_copy["GlobalSplitU"] + elif state["GlobalSplitU"] > 0: + state_copy["GlobalSplitU"] = "M" + state_copy["WorkGroupMapping"] = "M" + state_copy["WorkGroupMappingXCC"] = "M" + state_copy["WorkGroupMappingXCCGroup"] = "M" + state_copy["StaggerU"] = "M" + state_copy["StaggerUStride"] = "M" + state_copy["StaggerUMapping"] = "M" + state_copy["GlobalSplitUCoalesced"] = "M" + state_copy["GlobalSplitUWorkGroupMappingRoundRobin"] = "M" + return state_copy + + +def getNameFull(state, splitGSU: bool): + requiredParameters = {} + for key in state: + if key in list(validParameters.keys()): + requiredParameters[key] = True + if "MatrixInstM" in state: + # Use MIWaveGroup and MIWaveTile instead of WG and MT + requiredParameters["MIWaveTile"] = True + requiredParameters["ThreadTile"] = False + return getNameMin(state, requiredParameters, splitGSU) + + +@lru_cache(maxsize=None) +def getParameterNameAbbreviation( name: str ): + return ''.join(c for c in name if c.isupper()) + +@ lru_cache(maxsize=None) +def getPrimitiveParameterValueAbbreviation(key, value): + if isinstance(value, str): + return getParameterNameAbbreviation(value) + elif isinstance(value, bool): + return "1" if value else "0" + elif isinstance(value, int): + if value >= 0: + return "%u" % value + else: # -1 -> n1 + return "n%01u" % abs(value) + elif isinstance(value, ProblemType): # will need to deal with this + return str(value) + elif isinstance(value, float): + val1 = int(value) + val2 = int(round(value*100)) - int(value)*100 + if val2 > 0: + s = "%dp%s" % (val1,str(val2).zfill(2)) + else: + s = "%d" % (val1) + return s + + +def getParameterValueAbbreviation(key, value): + if key == "ISA": + return f"{value[0]}{value[1]}{value[2]:x}" + compositieTypes = (dict, list, tuple,) + if not isinstance(value, compositieTypes): + return getPrimitiveParameterValueAbbreviation(key, value) + elif isinstance(value, tuple): + return ''.join(str(v) for v in value) + elif isinstance(value, list): + return '_'.join(getParameterValueAbbreviation(key, v) for v in value) + elif isinstance(value, dict): + return "_".join(f"{pos:d}{k:d}" for pos,k in value.items()) + else: + raise Exception(f"Parameter {key}={value} is new object type ({type(value)})") + + +def getNameMin(state, requiredParameters, splitGSU: bool, ignoreInternalArgs = False): + if "CustomKernelName" in state and state["CustomKernelName"]: + return state["CustomKernelName"] + + components = [] + backup = state["ProblemType"]["GroupedGemm"] + if ignoreInternalArgs: + state["ProblemType"]["GroupedGemm"] = False + if "ProblemType" in state: + components.append(f'{str(state["ProblemType"])}') + # name += str(state["ProblemType"]) + "_" + if ignoreInternalArgs: + state["ProblemType"]["GroupedGemm"] = backup + if "MacroTile0" in state \ + and "MacroTile1" in state \ + and "DepthU" in state: + components.append(f'{getParameterNameAbbreviation("MacroTile")}{state["MacroTile0"]}x{state["MacroTile1"]}x{state["DepthU"]}') + if "MatrixInstM" in state: + components.append(f'{getParameterNameAbbreviation("MatrixInstruction")}{state["MatrixInstM"]}x{state["MatrixInstN"]}x{state["MatrixInstB"]}') + backup = state["GlobalSplitU"] + if ignoreInternalArgs: + if splitGSU: + state["GlobalSplitU"] = "M" if (state["GlobalSplitU"] > 1) else state["GlobalSplitU"] + elif state["GlobalSplitU"] > 0: + requiredParameters["GlobalSplitU"] = False + requiredParameters["WorkGroupMapping"] = False + requiredParameters["WorkGroupMappingXCC"] = False + requiredParameters["WorkGroupMappingXCCGroup"] = False + requiredParameters["StaggerU"] = False + requiredParameters["StaggerUStride"] = False + requiredParameters["StaggerUMapping"] = False + requiredParameters["GlobalSplitUCoalesced"] = False + requiredParameters["GlobalSplitUWorkGroupMappingRoundRobin"] = False + useWaveTile, useThreadTile = requiredParameters.get("MIWaveTile", False), requiredParameters.get("ThreadTile", False) + if 'MatrixInstM' in state: + requiredParameters["MIWaveTile"] = True + requiredParameters["ThreadTile"] = False + else: + requiredParameters["MIWaveTile"] = False + requiredParameters["ThreadTile"] = True + components.append('SN') + for key in sorted(state.keys()): + if key in requiredParameters and key[0] != '_': + if requiredParameters[key] and key != "CustomKernelName": + components.append(f'{getParameterNameAbbreviation(key)}{getParameterValueAbbreviation(key, state[key])}') + state["GlobalSplitU"] = backup + requiredParameters["GlobalSplitU"] = True + requiredParameters["WorkGroupMapping"] = True + requiredParameters["WorkGroupMappingXCC"] = True + requiredParameters["WorkGroupMappingXCCGroup"] = True + requiredParameters["StaggerU"] = True + requiredParameters["StaggerUStride"] = True + requiredParameters["StaggerUMapping"] = True + requiredParameters["GlobalSplitUCoalesced"] = True + requiredParameters["GlobalSplitUWorkGroupMappingRoundRobin"] = True + requiredParameters["MIWaveTile"] = useWaveTile + requiredParameters["ThreadTile"] = useThreadTile + return '_'.join(components) + + +def getSerialNaming(objs): + data = {} + for obj in objs: + for paramName in sorted(obj.keys()): + if paramName in validParameters.keys(): + paramValue = obj[paramName] + if paramName in data: + if paramValue not in data[paramName]: + data[paramName].append(paramValue) + else: + data[paramName] = [ paramValue ] + maxObjs = 1 + for paramName in data: + if not isinstance(data[paramName][0], dict): + data[paramName] = sorted(data[paramName]) + maxObjs *= len(data[paramName]) + numDigits = len(str(maxObjs)) + return [ data, numDigits ] + + +def getNameSerial(state, serialNaming): + data = serialNaming[0] + numDigits = serialNaming[1] + serial = 0 + multiplier = 1 + for paramName in sorted(state.keys()): + if paramName in list(validParameters.keys()): + paramValue = state[paramName] + paramData = data[paramName] + paramNameMultiplier = len(paramData) + if paramValue in paramData: + paramValueIdx = paramData.index(paramValue) + serial += paramValueIdx * multiplier + multiplier *= paramNameMultiplier + name = "%s%0*u" % ("S" if hasattr(state, "_state") else "K", \ + numDigits, serial) + return name + + +def shortenFileBase(kernelMinNaming, splitGSU, kernel): + base = getKernelName(kernelMinNaming, splitGSU, kernel) + if len(base) <= MAX_FILENAME_LENGTH: + return base + import hashlib + import base64 + pivot = MAX_FILENAME_LENGTH * 3 // 4 + firstPart = base[:pivot] + secondPart = base[pivot:] + secondHash = hashlib.sha256(secondPart.encode()).digest() + secondPart = base64.b64encode(secondHash, b'_-').decode() + return firstPart + secondPart + + +def getKernelFileBase(useShortNames: bool, splitGSU: bool, kernelMinNaming, kernelSerialNaming, kernel): + if "CustomKernelName" in kernel and kernel["CustomKernelName"]: + fileBase = kernel["CustomKernelName"] + elif useShortNames: + fileBase = getNameSerial(kernel, kernelSerialNaming) + else: + fileBase = shortenFileBase(kernelMinNaming, splitGSU, kernel) + return fileBase + + +def getKernelName(kernelMinNaming, splitGSU, kernel): + kernelName = getNameMin(kernel, kernelMinNaming, splitGSU, True) + return kernelName diff --git a/tensilelite/Tensile/SolutionStructs/Problem.py b/tensilelite/Tensile/SolutionStructs/Problem.py new file mode 100644 index 0000000000..f1c9ffba92 --- /dev/null +++ b/tensilelite/Tensile/SolutionStructs/Problem.py @@ -0,0 +1,1115 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +from collections import OrderedDict +from collections.abc import Mapping + +from typing import List + +from Tensile.TensileInstructions.Base import fastdeepcopy as deepcopy +from Tensile.Activation import ActivationType +from Tensile.TensileInstructions.DataType import DataType +from Tensile.Common.Constants import INDEX_CHARS +from Tensile.Common.Utilities import assignParameterWithDefault, printWarning, print2, print1, printExit + + + + +class ProblemSizeRange: + + def __init__(self, problemType, config): + self.totalIndices = 1+max(problemType["IndexAssignmentsA"]) + problemType["NumIndicesLD"] + if len(config) < self.totalIndices: + for i in range(len(config), self.totalIndices): + if i < self.totalIndices - problemType["NumIndicesLD"]: + config.append(0) + else: + config.append([0]) + + self.indexMax = [] + self.indexIsSized = [] + self.indicesSized = [] + self.indicesMapped = [] + for i in range(0, self.totalIndices): + dim = deepcopy(config[i]) + if isinstance(dim, list): + if len(dim) == 1: + self.indicesSized.append([dim[0], 1, 0, dim[0]]) + elif len(dim) == 2: + self.indicesSized.append([dim[0], dim[0], 0, dim[1]]) + elif len(dim) == 3: + self.indicesSized.append([dim[0], dim[1], 0, dim[2]]) + elif len(dim) == 4: + self.indicesSized.append([dim[0], dim[1], dim[2], dim[3]]) + else: + printExit("dimension[%u] config (%s) has %u descriptors rather than 1-4." + % ( i, dim, len(dim) )) + self.indexIsSized.append(True) + self.indexMax.append(self.indicesSized[len(self.indicesSized)-1][3]) + + elif isinstance(dim, int): + self.indicesMapped.append(dim) + self.indexIsSized.append(False) + self.indexMax.append(self.indicesSized[self.indicesMapped[ \ + len(self.indicesMapped)-1]][3]) + + # max num elements in each tensor + self.maxNumElements = [ 1, 1, 1 ] + for i in range(0, problemType["NumIndicesC"]): + self.maxNumElements[0] *= self.indexMax[i] + for i in problemType["IndexAssignmentsA"]: + self.maxNumElements[1] *= self.indexMax[i] + for i in problemType["IndexAssignmentsB"]: + self.maxNumElements[2] *= self.indexMax[i] + + self.totalProblemSizes = 1 + self.numProblemSizes = [] # per index + self.problemSizeToIndex = [] + self.problemIndexToSize = [] + sizedIdx = 0 + for i in range(0, len(self.indexIsSized)): + self.problemSizeToIndex.append({}) + self.problemIndexToSize.append({}) + if self.indexIsSized[i]: + self.numProblemSizes.append(0) + index = self.indicesSized[sizedIdx] + sizedIdx += 1 + currentSize = index[0] + currentIncrement = index[1] + while currentSize <= index[3]: + currentSize += currentIncrement + currentIncrement += index[2] + self.numProblemSizes[i] += 1 + else: + self.numProblemSizes.append(1) + self.totalProblemSizes *= self.numProblemSizes[i] + + ######################################## + # enumerate problem sizes + currentSizedIndexSizes = [] + currentSizedIndexIncrements = [] + for i in range(0, len(self.indicesSized)): + currentSizedIndexSizes.append(self.indicesSized[i][0]) + currentSizedIndexIncrements.append(self.indicesSized[i][1]) + + # iterate over all problem sizes + self.problemSizes = [] + moreProblemSizes = True + problemIdx = 0 + problemSize = [0]*self.totalIndices + while moreProblemSizes: + #/ convert current sized and mapped indices to full sizes + currentSizedIdx = 0 + currentMappedIdx = 0 + for i in range(0, self.totalIndices): + if self.indexIsSized[i]: + problemSize[i] = currentSizedIndexSizes[currentSizedIdx] + currentSizedIdx+=1 + else: + problemSize[i] = problemSize[self.indicesMapped[currentMappedIdx]] + currentMappedIdx+=1 + self.problemSizes.append(tuple(problemSize)) + + #/ increment sizes for next benchmark + currentSizedIndexSizes[0] += currentSizedIndexIncrements[0] + currentSizedIndexIncrements[0] += self.indicesSized[0][2] + for i in range(1, len(self.indicesSized)+1): + # if prior index past max, reset to min and increment next index + if currentSizedIndexSizes[i-1] > self.indicesSized[i-1][3]: + #/ reset prior index + currentSizedIndexSizes[i-1] = self.indicesSized[i-1][0] + currentSizedIndexIncrements[i-1] = self.indicesSized[i-1][1] + # increment next index + if i >= len(self.indicesSized): + moreProblemSizes = False + else: + currentSizedIndexSizes[i] += currentSizedIndexIncrements[i] + currentSizedIndexIncrements[i] += self.indicesSized[i][2] + + problemIdx+=1 + + ######################################## + # YAML format + def __str__(self): + state = "[ " + sizedIdx = 0 + mappedIdx = 0 + for i in range(0, len(self.indexIsSized)): + if self.indexIsSized[i]: + indices = self.indicesSized[sizedIdx] + state += "[ %u, %u, %u, %u ]" \ + % (indices[0], indices[1], indices[2], indices[3]) + sizedIdx += 1 + else: + indices = self.indicesSized[self.indicesMapped[mappedIdx]] + state += str(self.indicesMapped[mappedIdx]) + mappedIdx += 1 + if i < len(self.indexIsSized)-1: + state += ", " + state += " ]" + return state + +class Problem: + """ Problem sizes, strides, padding and other info""" + def __init__(self, sizes=None, stridesA=None, stridesB=None, stridesC=None, stridesD=None, count=None): + self.sizes = tuple(sizes) if sizes else None + self.stridesA = tuple(stridesA) if stridesA else None + self.stridesB = tuple(stridesB) if stridesB else None + self.stridesC = tuple(stridesC) if stridesC else None + self.stridesD = tuple(stridesD) if stridesD else None + + self.count = count + + def __str__(self): + rv= "{ sizes:" + str(list(self.sizes)) + if self.stridesA: + rv += ", stridesA:" + str(list(self.stridesA)) + if self.stridesB: + rv += ", stridesB:" + str(list(self.stridesB)) + if self.stridesC: + rv += ", stridesC:" + str(list(self.stridesC)) + if self.stridesD: + rv += ", stridesD:" + str(list(self.stridesD)) + rv += " }" + return rv + +class ExactList(Problem): + def __init__(self, e, problemType): + if len(e) == problemType["TotalIndices"]: + if -1 in e: + printExit("ExactSize %s contains -1" % (e)) + if problemType["OperationType"] == "GEMM": + e += [-1, -1, -1, -1] + e = ExactList.convertLeadingDims(problemType, tuple(e)) + sizes=e + + elif len(e) == (problemType["TotalIndices"] + problemType["NumIndicesLD"]): + sizes = ExactList.convertLeadingDims(problemType, tuple(e)) + else: + printExit("ExactSize %s doesn't match indices of ProblemType %s, totalIndices=%d, len e=%d, NumIndicesLD = %d" \ + % (e, problemType, problemType["TotalIndices"], len(e), problemType["NumIndicesLD"]) ) + + # TODO- pass strides here, remove calls to convertLeadingDims + Problem.__init__(self, sizes=sizes) + + def __str__(self): + return str(list(self.sizes)) + + @staticmethod + def convertLeadingDims(problemType, problemSize, stridesA = None, stridesB = None, stridesC = None, stridesD = None): + # FIXME-problem: refactor to eliminate max, pass strides in strideB parm rather than hacked + # onto the end of the sizes list + predStridesD = stridesD is not None and stridesD[1] != -1 + predStridesC = stridesC is not None and stridesC[1] != -1 + predStridesA = stridesA is not None and stridesA[1] != -1 + predStridesB = stridesB is not None and stridesB[1] != -1 + return problemSize[:problemType["NumIndicesC"]+1] + \ + (max(problemSize[0], problemSize[problemType["IndexAssignmentsLD"][0]]) if not predStridesD else stridesD[1], ) + \ + (max(problemSize[0], problemSize[problemType["IndexAssignmentsLD"][1]]) if not predStridesC else stridesC[1], ) + \ + (max(problemSize[problemType["IndexAssignmentsLD"][2]], + problemSize[problemType["IndexAssignmentsA"][0]]) if not predStridesA else stridesA[1], ) + \ + (max(problemSize[problemType["IndexAssignmentsLD"][3]], + problemSize[problemType["IndexAssignmentsB"][0]]) if not predStridesB else stridesB[1], ) + + +class ExactDict(Problem): + AllowedFields = [ 'count', 'sizes', 'stridesA', 'stridesB', 'stridesC', 'stridesD' ] + + def __init__(self, e, problemType): + Problem.__init__(self) + + for f in e: + if f in ExactDict.AllowedFields: + setattr(self, f, e[f]) + else: + raise RuntimeError ("specified field '%s' is not a valid Exact dict field"%f) + + if problemType: + if "OperationType" in problemType and problemType["OperationType"] == "GEMM": + sizesTuple = tuple(self.sizes + [-1, -1, -1, -1]) + self.sizes = ExactList.convertLeadingDims(problemType, sizesTuple, self.stridesA, self.stridesB, self.stridesC, self.stridesD) + + if problemType: + if "OperationType" in problemType and problemType["OperationType"] == "GEMM": + if len(self.sizes) != (problemType["TotalIndices"] + problemType["NumIndicesLD"]): + # FIXME-ExactDict size descriptor still (but preferrably not so) uses 8-tuple for GEMM problems + raise RuntimeError ("specified size=%s does not have enough indices for problem (expected %d, got %d)" \ + % (self.sizes, problemType["TotalIndices"]+problemType["NumIndicesLD"], len(self.sizes))) + elif len(self.sizes) != problemType["TotalIndices"]: + raise RuntimeError ("specified size=%s does not have enough indices for problem (expected %d, got %d)" \ + % (self.sizes, problemType["TotalIndices"], len(self.sizes))) + + +################################################################################ +# ProblemSizes +################################################################################ +""" +Adapter class for class `ProblemSizes`. It satisfies the implicit usage requirement +of ClientWriter.writeClientConfig() by converting ExactLogic to list of `Problem` objects +""" +class ProblemSizesMock: + def __init__(self, exactLogic): + self.problems = [Problem(problem) for problem, solution in exactLogic] + +class ProblemSizesMockDummy: + def __init__(self): + self.problems = [Problem(sizes=[128, 128, 1, 512])] + +class ProblemSizes: + + ######################################## + def __init__(self, problemType, config): + self.problemType = problemType + self.ranges = [] + self.exacts = [] + self.minStrides = None + if config: + for dictionary in config: + for sizeTypeKey in dictionary: + #print ("PROBLEM parsed:", sizeTypeKey, dictionary[sizeTypeKey]) + if sizeTypeKey == "Range": + psr = ProblemSizeRange(problemType, dictionary[sizeTypeKey]) + self.ranges.append( psr ) + elif sizeTypeKey == "Exact": + e= dictionary[sizeTypeKey] + if isinstance(e,list): + self.exacts.append(ExactList(e, problemType)) + elif isinstance(e,dict): + self.exacts.append(ExactDict(e, problemType)) + else: + printExit("Unsupported Exact type==%s"%type(e)) + elif sizeTypeKey == "MinStride": + e = dictionary[sizeTypeKey] + if len(e) != problemType["TotalIndices"]: + printExit("MinStride %s doesn't match indices of ProblemType %s" \ + % (e, problemType) ) + if self.minStrides: + printExit("Only one MinStride command is allowed in a ProblemsSizes definition. Previous minStrides:%s, New minstride:%s" \ + % (self.minStrides, e) ) + + self.minStrides=(tuple(e)) + else: + printExit("ProblemSize Type %s not supported"%sizeTypeKey) + + if not self.minStrides: + # set harmless default mins of 0 + self.minStrides = ([0]* problemType["TotalIndices"]) + + # not the ideal spot, but convert leading dims that are below the minimum size + if problemType["OperationType"] == "GEMM": + for i in range(0, len(self.ranges)): + self.ranges[i].problemSizes[:] = \ + [ExactList.convertLeadingDims(self.problemType, problemSize) for problemSize in self.ranges[i].problemSizes] + + self.problems = OrderedDict() + for sizeRange in self.ranges: + for rangeSize in sizeRange.problemSizes: + self.problems.update({Problem(rangeSize) : 1}) + for e in self.exacts: + self.problems.update({e : 1}) + self.problems = list(self.problems.keys()) + self.totalProblemSizes = len(self.problems) + + # max sizes + self.maxD = 0 + self.maxC = 0 + self.maxA = 0 + self.maxB = 0 + for problem in self.problems: + problemSize = problem.sizes # FIXME-problem. This should use problem.strides* + + sizeLdd = problemSize[self.problemType["IndexAssignmentsLD"][0]] if problemType["OperationType"] == "GEMM" else problemSize[0] + sizeD = max(self.minStrides[0], sizeLdd) + for i in range(1, problemType["NumIndicesC"]): + sizeD *= max(self.minStrides[i], problemSize[i]) + + sizeLdc = problemSize[self.problemType["IndexAssignmentsLD"][1]] if problemType["OperationType"] == "GEMM" else problemSize[0] + sizeC = max(self.minStrides[0], sizeLdc) + for i in range(1, problemType["NumIndicesC"]): + sizeC *= max(self.minStrides[i], problemSize[i]) + + sizeLda = problemSize[self.problemType["IndexAssignmentsLD"][2]] \ + if problemType["OperationType"] == "GEMM" \ + else problemSize[self.problemType["IndexAssignmentsA"][0]] + sizeA = max(self.minStrides[self.problemType["IndexAssignmentsA"][0]], sizeLda) + for i in self.problemType["IndexAssignmentsA"][1:]: + sizeA *= max(self.minStrides[i], problemSize[i]) + + sizeLdb = problemSize[self.problemType["IndexAssignmentsLD"][3]] \ + if problemType["OperationType"] == "GEMM" \ + else problemSize[self.problemType["IndexAssignmentsB"][0]] + sizeB = max(self.minStrides[self.problemType["IndexAssignmentsB"][0]], sizeLdb) + for i in self.problemType["IndexAssignmentsB"][1:]: + sizeB *= max(self.minStrides[i], problemSize[i]) + + self.maxD = max(self.maxD, sizeD) + self.maxC = max(self.maxC, sizeC) + self.maxA = max(self.maxA, sizeA) + self.maxB = max(self.maxB, sizeB) + + def __str__(self): + s = "ProblemSizes\n" + for sizeRange in self.ranges: + s += " %s" % sizeRange + return s + +################################################################################ +# ProblemType +# name of solution should begin with name of problemType, and arguments can be listed out explicitly + +################################################################################ +# Default Problem Type +################################################################################ +_defaultProblemType = { + # =GEMM uses TransposeA,B parameters and makes the problem type more readable for users + # =TensorContraction requires specifying + "OperationType": "GEMM", # GEMM, TensorContraction, ConvolutionForward, ConvolutionBackwardData, ConvolutionBackwardWeights + "DataType": 0, # data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeA": 0, # A data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeB": 0, # B data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeE": 0, # E data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DataTypeAmaxD": 0, # AmaxD data type can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "DestDataType": 0, # destination data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "ComputeDataType": 0, # compute data types can specified by a variety of ways, such as "s", as listed in SolutionStructs.py::DataType + "F32XdlMathOp": 0, # reducing intermediate precision from f32 to a specific type, such as "x", as listed in SolutionStructs.py::DataType. + # in:f32, intermediate:xf32, out:f32. f32 = xf32(f32) * xf32(f32) + "UseBeta": True, # =True use beta parameter (asm will check for B=0 and optimize the write for that), =False don't use beta parameter + "UseE": False, # =True use output E to output gemm results before activation + "Gradient": False, # =True set globalWriteElements to gradient mode + "UseBias": 0, # =1 support bias vector on M direction, =2 support bias vector on N direction, =3 support bias vector on both M,N direction + "BiasSrc": "D", # This parameter is used in gradient + bias. Support A, B, D. + "UseScaleAB": "", # Support "", "Scalar", and "Vector" + "UseScaleCD": False, # =True use scaleC, scaleD + "UseScaleAlphaVec": 0, # =1 support alpha vector on M direction, =2 support bias vector on N direction, =3 support alpha vector on both M,N direction + "HighPrecisionAccumulate": False, # f32 += f16*f16 + "SilentHighPrecisionAccumulate": False, # Keep kernel names the same for HPA mode. Useful for testing. + "Sparse": 0, # 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B + "ComplexConjugateA": False, # complex data should be conjugated for "C" transpose case + "ComplexConjugateB": False, + "StochasticRounding": False, # By default, IEEE RNE rounding + # for OperationType == GEMM + "TransposeA": False, # =True means transA="T" or "C", =False means transA = "N" + "TransposeB": True, + "Batched": False, # add batching dimension + "StridedBatched": True, # use to select general batch or strided batch + "GroupedGemm": False, # use to select general batch or strided batch + # for OperationType == TensorContraction + # - Indices < NumIndicesC are Free or Batch indices and appear in C and D + # - Indices which appear in both A and B, and are < NumIndicesC are batch. A and B must have same number of batch indices. + # - Indices which appear in both A and B, and are >= NumIndicesC are summation. A and B must have same number of summation indices. + # - Indices which appear in A or B (but not both), are Free. A and B may have different numbers of free indices. + # - Summation loops are nested from smallest index number to largest, with the largest summation index as the 'unroll' loop. + # - Memory order of C and D matrices is always 0..NumIndicesC-1, with 0 as the fastest-moving. + # - By choosing index assignments the output can be 'transposed'. For example if IA=[1,2] IB=[0,2] then 0 is the coalesced dim for C/D. + # - Likewise batch index may be assigned between two free indices to control the output order, ie to write in CNHW format. + # - For example : IA=[0,1,3] IB=[2,1,3]. 0,2 are free indices; 1 is batch. + "IndexAssignmentsA": [0, 2], + "IndexAssignmentsB": [1, 2], + "NumIndicesC": 2, + # use initial strides for AB. + # This has some performance impact for the increased flexibility: + # - Additional strides will be passed into the kernel and will occupy SGPR registers + # - GlobalReadWidth must be 1 (since elements are not guaranteed to be adjacent in memory) + "UseInitialStridesAB": False, + # use initial strides for CD. + # This has some performance impact for the increased flexibility: + # - Additional strides will be passed into the kernel and will occupy SGPR registers + # - Additional multiply on the store address path + # -VectorStore must be 0. If VectorStore is -1, it will be silently set to 0 internally. + "UseInitialStridesCD": False, + "AllowNoFreeDims": False, # allow A or B to specify no free dims + # (if false, A and B must have at least one free dim) + # (if true, A and B must have at least one free or batch dim) + # SetConstStride* sets the specified stride in the problem. + # These no longer generate predicates - see AssertStrideEqualA/B below + # List of pairs of [index, constValue]. + # Index is a member of the global index assignments (not an offset into IndexAssignmentsA/B) + # EX: SetConstStrideA: [ [3, 1], [2, 4] ] sets + # strideA for index3 to constant '1' and stride for index2 to constant '4'. + "SetConstStrideA": [], + "SetConstStrideB": [], + "SetConstStrideBias": [], + # Summation dimension indices + "MirrorDimsA": [], + "MirrorDimsB": [], + "MirrorDimsMetadata": [], + # for LD description + "NumIndicesLD": 4, + "IndexAssignmentsLD": [3, 4, 5, 6], # order is LDD, LDC, LDA, LDB + # Tile aware solution selection + "TileAwareSelection": False, + # Activation + "Activation": False, + "ActivationNoGuard": False, + # AmaxD + "OutputAmaxD": False, + # For kernels putting arguments in workspaces instead of kernel arguments, they can choose to support user arguments input instead. + "SupportUserArgs": True, + "SwizzleTensorA": False, + "SwizzleTensorB": False, +} + +# The supported typed GEMM, each entry is (Ti, To, Tc). +# DataType (Ti) = The data-type of the input matrices: A/B +# DestDataType (To) = The data-type of the output matrices: C/D +# ComputeDataType (Tc) = The data-type of computation: alpha/beta: +# Cinternal: basically should == ComputeDataType +# This is used in _checkIfSupportedGEMMType() +_validGEMMTypes = [ + ("H", "H", "H"), + ("S", "S", "S"), + ("D", "D", "D"), + ("C", "C", "C"), + ("Z", "Z", "Z"), + ("H", "H", "S"), + ("H", "S", "S"), + ("B", "B", "S"), + ("B", "S", "S"), + ("B", "H", "S"), + ("I8", "I", "I"), + ("4xi8", "I", "I"), + ("I8", "I8", "I"), + ("I8", "I", "S"), + ("I8", "I8", "S"), + ("I8", "H", "S"), + ("I8", "B", "S"), + ("F8", "S", "S"), + ("B8", "S", "S"), + ("F8B8", "S", "S"), + ("B8F8", "S", "S"), + ("F8", "H", "S"), + ("B8", "H", "S"), + ("F8B8", "H", "S"), + ("B8F8", "H", "S"), + ("B8", "B", "S"), + ("H", "F8", "S"), + ("F8", "B", "S"), + ("F8B8", "B", "S"), + ("B8F8", "B", "S"), # in/out are both R8 + ("F8", "F8", "S"), + ("B8", "B8", "S"), + ("F8B8", "B8", "S"), + ("B8F8", "B8", "S"), + ("F8", "B8", "S"), + ("B8", "F8", "S"), + ("F8B8", "F8", "S"), + ("B8F8", "F8", "S"), # F8 NANOO + ("F8N", "S", "S"), + ("B8N", "S", "S"), + ("F8B8N", "S", "S"), + ("B8F8N", "S", "S"), + ("F8N", "H", "S"), + ("B8N", "H", "S"), + ("F8B8N", "H", "S"), + ("B8F8N", "H", "S"), + ("B8N", "B", "S"), + ("H", "F8N", "S"), + ("F8N", "B", "S"), + ("F8B8N", "B", "S"), + ("B8F8N", "B", "S"), # in/out are both R8 + ("F8N", "F8N", "S"), + ("B8N", "B8N", "S"), + ("F8B8N", "B8N", "S"), + ("B8F8N", "B8N", "S"), + ("F8N", "B8N", "S"), + ("B8N", "F8N", "S"), + ("F8B8N", "F8N", "S"), + ("B8F8N", "F8N", "S"), +] + + +# All HPA types are listed here (HPA=T). The name of the library logic files for these types is: +# *_TiToTc_BH*.yaml where Ti, To, and Tc are the data types of A/B, C/D, and computation, respectively. +# The name of the library logic files for non-HPA (HPA=F) types is: *_TiB*.yaml. +_HPATypes = [ + ("H", "S", "S"), + ("H", "H", "S"), + ("B", "B", "S"), + ("B", "S", "S"), + ("B", "H", "S"), + ("I8", "I", "I"), + ("4xi8", "I", "I"), + ("I8", "I", "S"), + ("I8", "I8", "S"), + ("I8", "H", "S"), + ("I8", "B", "S"), + ("F8", "S", "S"), + ("B8", "S", "S"), + ("F8B8", "S", "S"), + ("B8F8", "S", "S"), + ("F8", "H", "S"), + ("B8", "H", "S"), + ("F8B8", "H", "S"), + ("B8F8", "H", "S"), + ("H", "F8", "S"), + ("F8", "B", "S"), + ("F8B8", "B", "S"), # in/out are both R8 + ("F8", "F8", "S"), + ("B8", "B8", "S"), + ("F8B8", "B8", "S"), + ("B8F8", "B8", "S"), + ("F8", "B8", "S"), + ("B8", "F8", "S"), + ("F8B8", "F8", "S"), + ("B8F8", "F8", "S"), + ("F8N", "S", "S"), + ("B8N", "S", "S"), + ("F8B8N", "S", "S"), + ("B8F8N", "S", "S"), + ("F8N", "H", "S"), + ("B8N", "H", "S"), + ("F8B8N", "H", "S"), + ("B8F8N", "H", "S"), + ("H", "F8N", "S"), + ("F8N", "B", "S"), + ("F8B8N", "B", "S"), # in/out are both R8 + ("F8N", "F8N", "S"), + ("B8N", "B8N", "S"), + ("F8B8N", "B8N", "S"), + ("B8F8N", "B8N", "S"), + ("F8N", "B8N", "S"), + ("B8N", "F8N", "S"), + ("F8B8N", "F8N", "S"), + ("B8F8N", "F8N", "S"), +] + + +class ProblemType(Mapping): + ######################################## + + @classmethod + def FromDefaultConfig(printIndexAssignmentInfo: bool): + return ProblemType(_defaultProblemType, printIndexAssignmentInfo) + + def __init__(self, config, printIndexAssignmentInfo: bool): + self.state = {} + + for key in _defaultProblemType: + assignParameterWithDefault(self.state, key, config, _defaultProblemType) + + # adjusting all data types + if "DataType" in config: + self["DataType"] = DataType(config["DataType"]) + self["DataTypeA"] = self["DataType"] + self["DataTypeB"] = self["DataType"] + else: + raise Exception("NO data type specified") + self["DataType"] = DataType(0) + self["DataTypeA"] = DataType(0) + self["DataTypeB"] = DataType(0) + + if "DataTypeA" in config: + self["DataTypeA"] = DataType(config["DataTypeA"]) + + if "DataTypeB" in config: + self["DataTypeB"] = DataType(config["DataTypeB"]) + + if "DestDataType" in config: + self["DestDataType"] = DataType(config["DestDataType"]) + else: + if "DataType" in config: + self["DestDataType"] = DataType(config["DataType"]) + else: + raise Exception("NO dest data type or data type specified") + self["DataType"] = DataType(0) + + self["DataTypeE"] = self["DestDataType"] + if "DataTypeE" in config: + self["DataTypeE"] = DataType(config["DataTypeE"]) + + if "ComputeDataType" in config: + self["ComputeDataType"] = DataType(config["ComputeDataType"]) + else: + if "DestDataType" in config: + self["ComputeDataType"] = DataType(config["DestDataType"]) + else: + if "DataType" in config: + self["ComputeDataType"] = DataType(config["DataType"]) + else: + raise Exception("NO compute data type, or dest data type, or data type specified") + self["DataType"] = DataType(0) + + # Just like DataTypeE is DestDataType by default; DataTypeAmaxD if ComputeDataType by default. + # So far we don't have to set it in config yamls + self["DataTypeAmaxD"] = self["ComputeDataType"] + if "DataTypeAmaxD" in config: + self["DataTypeAmaxD"] = DataType(config["DataTypeAmaxD"]) + + if self["Sparse"]: + self["DataTypeMetadata"] = DataType("I8") + + if "F32XdlMathOp" in config: + self["F32XdlMathOp"] = DataType(config["F32XdlMathOp"]) + else: + self["F32XdlMathOp"] = DataType(0) + + # Modifying ComputeDataType for HHH+HPA: if (HHH+HPA), convert it to HHS_BH by setting ComputeDataType to S. + if self["ComputeDataType"].isHalf() and self["DataType"].isHalf() and self["HighPrecisionAccumulate"]: + printWarning("Inconsistent DataTypes: DataType == f16, DestType == f16, ComputeDataType == f16, but HPA == True (HHH+HPA, no such a type); Converting HHH+HPA to HHS_BH by setting compute data type to f32.") + self["ComputeDataType"] = DataType('s') + + # Modifying ComputeDataType for BBB+HPA: if (BBB+HPA), convert it to BBS_BH by setting ComputeDataType to S. + if self["ComputeDataType"].isBFloat16() and self["DataType"].isBFloat16() and self["HighPrecisionAccumulate"]: + printWarning("Inconsistent DataTypes: DataType == bf16, DestType == bf16, ComputeDataType == bf16, but HPA == True (BBB+HPA, no such a type); Converting BBB+HPA to BBS_BH by setting compute data type to f32.") + self["ComputeDataType"] = DataType('s') + + # Modifying ComputeDataType for I8I8I_BH: if (I8I8I8+HPA), convert it to I8I8I_BH by setting ComputeDataType to i. + if self["ComputeDataType"].isInt8() and DataType(config["DataType"]).isInt8() and self["HighPrecisionAccumulate"]: + print2("DataType == i8 and HPA == True; setting compute data type to int32") + self["ComputeDataType"] = DataType('i') + + if self["OperationType"] == "GEMM": + self._checkIfSupportedGEMMType() + self.initGEMM() + else: + raise Exception("Unsupported OperationType = %s" % self["OperationType"]) + + self.state["AssignedDerivedParameters"] = False + ProblemType.assignDerivedParameters(self.state, printIndexAssignmentInfo) + + for tc in ('A', 'B'): + for sc in self["SetConstStride%s"%tc] : + (anchorDim, stride) = sc[:2] + if anchorDim not in self.state["IndexAssignments%s"%tc]: + raise Exception("SetConstStride%s=%s anchorDim=%u is not in IndexAssignments%s"%(tc, sc, anchorDim, tc)) + + # Bias + # If compute data type is not equal to dest data type, tensile will run conversion kernel. + # In this case we don't need to apply bias in beta only kernel. + if "UseBias" in config: + if self["ComputeDataType"] != self["DestDataType"]: + self["BetaOnlyUseBias"] = False + else: + self["BetaOnlyUseBias"] = True if self["UseBias"] > 0 else False + if "BiasDataTypeList" in config: + self["BiasDataTypeList"] = [DataType(btype) for btype in config["BiasDataTypeList"]] + self["BiasDataTypeList"].sort() # Make name unique + else: + self["BiasDataTypeList"] = getBiasDataTypeListDefault(self) + else: + self["BetaOnlyUseBias"] = False + self["BiasDataTypeList"] = [] + + # Activation + # Currently, ActivationType supports only 'all' and 'hipblaslt_all', and is active only when the Activation configuration is set to True. + # Otherwise, ActivationType will be set to 'none'. + if "Activation" in config: + typeStr = config.get("ActivationType", 'none') + if typeStr not in ['all', 'hipblaslt_all']: + typeStr = 'none' + else: + typeStr = 'none' + self["ActivationType"] = ActivationType(typeStr) + if "ActivationComputeDataType" in config: + self["ActivationComputeDataType"] = DataType(config["ActivationComputeDataType"]) + else: + self["ActivationComputeDataType"] = self["ComputeDataType"] + + if self["ActivationType"] != 'none': + # This is a dummy guard in case we currently don't have a converter to convert data from compute type to activation compute type + if self["ActivationComputeDataType"] not in [self["ComputeDataType"], self["DestDataType"]]: + printWarning("TensileLite currently only supports ActivationComputeDataType (%s) = ComputeDataType (%s) or DestDataType (%s). \ + ActivationComputeDataType will be set to ComputeDataType automatically."%(self["ActivationComputeDataType"].toChar(), \ + self["ComputeDataType"], \ + self["DestDataType"])) + self["ActivationComputeDataType"] = self["ComputeDataType"] + if (self["ActivationComputeDataType"].numRegisters() != self["ComputeDataType"].numRegisters()) and \ + (self["DataType"].numRegisters() < self["DestDataType"].numRegisters()): + printWarning("TensileLite only supports ActivationComputeDataType = ComputeDataType if DestDataType > DataType. \ + ActivationComputeDataType will be set to ComputeDataType automatically.") + self["ActivationComputeDataType"] = self["ComputeDataType"] + + if "UseE" in config: + if config["UseE"]: + if self["ActivationType"] == 'none': + printWarning("Use E is disabled cause Activation is set to False.") + self["UseE"] = False + else: + self["UseE"] = config["UseE"] + else: + self["UseE"] = config["UseE"] + + if "Gradient" in config: + if config["Gradient"]: + if (not self["UseBias"]) and self["ActivationType"] == 'none': + printWarning("Gradient is disabled cause bias and activation are both disabled.") + self["Gradient"] = False + if self["ActivationType"] != 'none' and self["UseE"] == False: + printWarning("Use E is enabled cause Activation is enabled.") + self["UseE"] = True + elif self["ActivationType"] != 'none' and self["UseE"] == False: + printWarning("Use E is disabled cause Activation is disabled.") + self["UseE"] = False + # if self["UseScaleAlphaVec"]: + # printWarning("Use scaleAlphaVec is disabled cause Gradient is enabled.") + # self["UseScaleAlphaVec"] = False + self["Gradient"] = config["Gradient"] + + # Need gradient info + biasSrcList = ["A", "B", "D"] + if "BiasSrc" in config: + if not self["Gradient"] and config["BiasSrc"] != "D": + printWarning("BiasSrc is set to D cause Gradient is disabled.") + self["BiasSrc"] = "D" + elif self["Gradient"]: + # # Currently only supports D :) + # if config["BiasSrc"] != "D": + # raise Exception("BiasSrc currently only supports D.") + if config["BiasSrc"] not in biasSrcList: + raise Exception("BiasSrc only supports A, B, D.") + + if "ActivationNoGuard" in config: + self["ActivationNoGuard"] = config["ActivationNoGuard"] + if self["ActivationNoGuard"]: + if self["ActivationType"] == 'none': + printWarning("ActivationNoGuard is set to False cause Acivation is off.") + self["ActivationNoGuard"] = False + if (not self["Gradient"]): + printWarning("ActivationNoGuard is set to False cause Gradient is off.") + self["ActivationNoGuard"] = False + + ################################################################################ + # Function checkIfSupportedGEMMType: + # Assures 3 data-types are valid, supported and well-assigned + # See the discussion in ValidParameters.py for validGEMMTypes + ################################################################################ + def _checkIfSupportedGEMMType(self): + inType = self["DataType"] + outType = self["DestDataType"] + computeType = self["ComputeDataType"] + + gemmType = ( inType.toChar(), outType.toChar(), computeType.toChar() ) + if gemmType not in _validGEMMTypes: + raise Exception("This typed-GEMM (Ti, To, Tc) = (%s, %s, %s) is not supported yet."%(gemmType[0], gemmType[1], gemmType[2])) + + ######################################## + def initGEMM(self): + sumIdx = 3 if self["Batched"] else 2 + self["IndexAssignmentsA"] = [0, sumIdx] # N + self["IndexAssignmentsB"] = [sumIdx, 1] # N + if self.state["Sparse"] == 2: + self["IndexAssignmentsMetadata"] = [sumIdx, 1] # N (ref B) + else: + self["IndexAssignmentsMetadata"] = [sumIdx, 0] # T (ref A) + if self["TransposeA"]: + self["IndexAssignmentsA"] = [sumIdx, 0] # T + if self["TransposeB"]: + self["IndexAssignmentsB"] = [1, sumIdx] # T + if self["Batched"]: + self["IndexAssignmentsA"].append(2) + self["IndexAssignmentsB"].append(2) + self["IndexAssignmentsMetadata"].append(2) + self["NumIndicesC"] = 3 + else: + self["NumIndicesC"] = 2 + + self["NumIndicesLD"] = 4 + self["IndexAssignmentsLD"][0] = self["NumIndicesC"] + 1 + for i in range(1, len(self["IndexAssignmentsLD"])): + self["IndexAssignmentsLD"][i] = self["IndexAssignmentsLD"][i-1] + 1 + + ######################################## + def isGEMM(self): + return self.operationType == 0 + + ######################################## + # determine d0, d1, dU + @staticmethod + def assignDerivedParameters(state, printIndexAssignmentInfo: bool=False): + if "AssignedDerivedParameters" in state: + if state["AssignedDerivedParameters"]: + return + state["AssignedDerivedParameters"] = False + + state["TotalIndices"] = max(max(state["IndexAssignmentsA"])+1, \ + max(state["IndexAssignmentsB"])+1) + + # determine num free, batch + state["IndicesFree"] = [] + state["IndicesBatch"] = [] + state["IndicesSummation"] = [] + + for i in range(0, state["NumIndicesC"]): + inA = i in state["IndexAssignmentsA"] + inB = i in state["IndexAssignmentsB"] + if inA and inB: + state["IndicesBatch"].append(i) + + elif inA or inB: + state["IndicesFree"].append(i) + else: + raise Exception("invalid index %u (inC but not (inA or inB))" % i) + + # determine num summation + for i in range(state["NumIndicesC"], state["TotalIndices"]): + inA = i in state["IndexAssignmentsA"] + inB = i in state["IndexAssignmentsB"] + if inA and inB: + state["IndicesSummation"].append(i) + else: + raise Exception("invalid index %u (expected summation but not (inA and inB))" % i) + # print index assignments + if printIndexAssignmentInfo: + print("IndicesFree: %s" % state["IndicesFree"]) + print("IndicesBatch: %s" % state["IndicesBatch"]) + print("IndicesSum: %s" % state["IndicesSummation"]) + print("IndexAssignmentsA: %s" % state["IndexAssignmentsA"]) + print("IndexAssignmentsB: %s" % state["IndexAssignmentsB"]) + print("NumIndicesC: %s" % state["NumIndicesC"]) + + for k in ('IndexAssignmentsA','IndexAssignmentsB'): + if len(state[k]) != len(set(state[k])): + raise Exception("duplicate index in %s=%s"% (k,state[k])) + + state["NumIndicesFree"] = len(state["IndicesFree"]) + state["NumIndicesBatch"] = len(state["IndicesBatch"]) + state["NumIndicesSummation"] = len(state["IndicesSummation"]) + if not state["AllowNoFreeDims"] and state["NumIndicesFree"] < 2 : + raise Exception("Tensile requires >= 2 free indices or set AllowNoFreeDims; FreeIndices=%s."% state["IndicesFree"]) + + # by default, unroll index will be the last/inner summation index + state["IndexUnroll"] = state["IndicesSummation"][len(state["IndicesSummation"])-1] + for i in range(0, len(state["IndexAssignmentsA"])): + if state["IndexAssignmentsA"][i] == state["IndexUnroll"]: + state["IndexUnrollA"] = i + break + for i in range(0, len(state["IndexAssignmentsB"])): + if state["IndexAssignmentsB"][i] == state["IndexUnroll"]: + state["IndexUnrollB"] = i + break + for i in range(0, len(state["IndexAssignmentsMetadata"])): + if state["IndexAssignmentsMetadata"][i] == state["IndexUnroll"]: + state["IndexUnrollM"] = i + break + #print2("IndexUnrollA: %u" % state["IndexUnrollA"]) + #print2("IndexUnrollB: %u" % state["IndexUnrollB"]) + + # assign d0, d1 + if state["AllowNoFreeDims"]: + dimList = state["IndicesFree"] + state["IndicesBatch"] + else: + dimList = state["IndicesFree"] + state["Index01A"] = [i for i in state["IndexAssignmentsA"] if i in dimList][0] + state["Index01B"] = [i for i in state["IndexAssignmentsB"] if i in dimList][0] + #print2("Index01A: %u" % state["Index01A"]) + #print2("Index01B: %u" % state["Index01B"]) + # Store code is optimized for 0 as the fastest-moving in memory + # whichever has lower stride in C (lower value), is 0, other is 1 + if state["Index01A"] < state["Index01B"]: + state["Index0"] = state["Index01A"] + state["Index1"] = state["Index01B"] + state["Tensor0"] = 0 + state["Tensor1"] = 1 + state["TileA"] = 0 + state["TileB"] = 1 + else: + state["Index0"] = state["Index01B"] + state["Index1"] = state["Index01A"] + state["Tensor0"] = 1 + state["Tensor1"] = 0 + state["TileA"] = 1 + state["TileB"] = 0 + + # generalize transpose + strideIdxA = state["IndexAssignmentsA"].index(state["Index01A"]) + strideIdxB = state["IndexAssignmentsB"].index(state["Index01B"]) + unrollIdxA = state["IndexAssignmentsA"].index(state["IndexUnroll"]) + unrollIdxB = state["IndexAssignmentsB"].index(state["IndexUnroll"]) + state["TLUA"] = strideIdxA < unrollIdxA + state["TLUB"] = strideIdxB < unrollIdxB + #state["TLUB"] = True # hack + + if printIndexAssignmentInfo: + print("TLUA: %s (stridePosA(%d) 1 : + name += "_FD%s"%("N" if factorDim == 2 else "MN") + + if self["UseE"]: + if self["Gradient"]: + name += "_Grad%s"%self["DataTypeE"].toChar() + else: + name += "_Aux%s"%self["DataTypeE"].toChar() # Not showing aux types + if self["OutputAmaxD"]: + name += "_AmaxD" + if self["Sparse"]: + if self["Sparse"] == 2: + name += "_SPB" + else: + name += "_SPA" + + # precision and other + # name += "_SB" if self["StridedBatched"] else "_GB" + if self["GroupedGemm"]: + name += "_GG" + else: + name += "" if self["StridedBatched"] else "_GB" # legacy + + # Activation Naming + if self["ActivationType"] != 'none': + if self["ActivationType"] == 'all': + name += "_A" + elif self["ActivationType"] == 'hipblaslt_all': + name += "_HA" + else: + name += "_%s"%str(self["ActivationType"]).upper() + name += self["ActivationComputeDataType"].toChar() + if self["ActivationNoGuard"]: name += "NG" + + if self["UseScaleAB"] == "Scalar": name += "_SAB" + elif self["UseScaleAB"] == "Vector": name += "_SABV" + if self["UseScaleCD"]: name += "_SCD" + if self["UseScaleAlphaVec"]: name += "_SAV" + + if self["SupportUserArgs"]: name += "_UserArgs" + + return name + + def keys(self): + return list(self.state.keys()) + def __len__(self): + return len(self.state) + def __iter__(self): + return iter(self.state) + def __getitem__(self, key): + return self.state[key] + def __setitem__(self, key, value): + self.state[key] = value + def __repr__(self): + return self.__str__() + def getAttributes(self): + return self.state + def __hash__(self): + return hash(str(self)) + def __eq__(self, other): + return isinstance(other, ProblemType) and self.getAttributes() == other.getAttributes() + def __ne__(self, other): + result = self.__eq__(other) + if result is NotImplemented: + return result + return not result + + def get(self, key, default=None): + try: + return self.state[key] + except: + return default + +################################################################################ +# Bias Type +################################################################################ + +def getBiasDataTypeListDefault(problem: ProblemType) -> List[DataType]: + bList = [] + for d in ["DataType", "ComputeDataType", "DestDataType"]: + dtype = DataType(problem[d]) + # filter out int8, because it is not supported by bias datatype + # TODO + if not dtype.isInt8(): + bList.append(dtype) + + biasDataTypeList = list(set(bList)) + biasDataTypeList.sort() # Make name unique + return biasDataTypeList + + diff --git a/tensilelite/Tensile/SolutionStructs.py b/tensilelite/Tensile/SolutionStructs/Solution.py similarity index 59% rename from tensilelite/Tensile/SolutionStructs.py rename to tensilelite/Tensile/SolutionStructs/Solution.py index df349212a5..ace3faa48d 100644 --- a/tensilelite/Tensile/SolutionStructs.py +++ b/tensilelite/Tensile/SolutionStructs/Solution.py @@ -22,64 +22,44 @@ # ################################################################################ -from .TensileInstructions import DataType, roundUpToNearestMultiple -from .TensileInstructions.Base import fastdeepcopy as deepcopy - -from .KernelWriterBetaOnly import KernelWriterBetaOnly -from .KernelWriterConversion import KernelWriterConversion -from .KernelWriterActivationEnumHeader import KernelWriterActivationEnumHeader -from .KernelWriterActivationFunction import KernelWriterActivationFunction -from .KernelWriterActivationOnly import KernelWriterActivationOnly -from .KernelWriterReduction import KernelWriterReduction - -from .AsmStoreState import VectorDataTypes -from .Activation import ActivationType +from Tensile.TensileInstructions import DataType, roundUpToNearestMultiple +from Tensile.TensileInstructions.Base import fastdeepcopy as deepcopy + +from Tensile.KernelWriterBetaOnly import KernelWriterBetaOnly +from Tensile.KernelWriterConversion import KernelWriterConversion +from Tensile.KernelWriterActivationEnumHeader import KernelWriterActivationEnumHeader +from Tensile.KernelWriterActivationFunction import KernelWriterActivationFunction +from Tensile.KernelWriterActivationOnly import KernelWriterActivationOnly +from Tensile.KernelWriterReduction import KernelWriterReduction + +from Tensile.Activation import ActivationType +from Tensile.AsmStoreState import VectorDataTypes + +from Tensile.CustomKernels import isCustomKernelConfig +from Tensile.Common import assignParameterWithDefault, IsaInfo, \ + defaultSolution, \ + defaultInternalSupportParams, \ + internalParameters, \ + print1, print2, printExit, printWarning, \ + roundUp, INDEX_CHARS, IsaVersion, SemanticVersion, \ + DepthUConfig +from Tensile.SolutionStructs.Naming import getNameFull +from Tensile.SolutionStructs.Problem import ProblemType +from Tensile.Toolchain.Component import Assembler -from .CustomKernels import isCustomKernelConfig +from .Utilities import reject -from .Common import assignParameterWithDefault, \ - defaultProblemType, defaultSolution, \ - defaultInternalSupportParams, \ - globalParameters, internalParameters, \ - print2, printExit, printWarning, \ - validMFMA, validSMFMA, validParameters, \ - validGEMMTypes, HPATypes, roundUp, validWMMA, INDEX_CHARS from collections import OrderedDict from collections.abc import Mapping from enum import Enum -from functools import lru_cache -from typing import List +from typing import List, Dict import collections import math -import operator import sys -######################################## -# Print a reject message : -def reject(state, *args): - if state and "NoReject" in state and state["NoReject"]: - return - - if globalParameters["PrintSolutionRejectionReason"]: - sys.stdout.write("\nreject: ") - for a in args: - print(a) - #traceback.print_stack(None, 2) - solutionIndex = state["SolutionIndex"] if (state != None and "SolutionIndex" in state) else -1 - if solutionIndex != -1: - # If we have valid solutionIndex, this means we are during TensileCreateLibrary stage - # In this stage, all solutions in the logic should be valid - # So if any rejection happens, print the warning for further check - # This will be done only when --global-parameters=PrintSolutionRejectionReason=True - solutionNameMin = state["SolutionNameMin"] if ("SolutionNameMin" in state) else None - # if we don't have SolutionNameMin, we simply use the problemTypeName - solutionNameMin = str(state["ProblemType"]) if (solutionNameMin == None) else solutionNameMin - print("!! Warning: Any rejection of a LibraryLogic is not expected, please check. \ - SolutionIndex: %d (or SolutionName/ProblemType: %s)"%(solutionIndex, solutionNameMin)) - if state != None: - state["Valid"] = False + # print a labled variable def pvar(state, field): @@ -93,852 +73,10 @@ class Fbs(Enum): Batch=1 # Expect to be batch dimension Sum=2 # Expect to be summation dimension -################################################################################ -# ProblemType -# name of solution should begin with name of problemType, and arguments can be listed out explicitly -class ProblemType(Mapping): - ######################################## - def __init__(self, config): - self.state = {} - - for key in defaultProblemType: - assignParameterWithDefault(self.state, key, config, defaultProblemType) - - # adjusting all data types - if "DataType" in config: - self["DataType"] = DataType(config["DataType"]) - self["DataTypeA"] = self["DataType"] - self["DataTypeB"] = self["DataType"] - else: - printExit("NO data type specified") - self["DataType"] = DataType(0) - self["DataTypeA"] = DataType(0) - self["DataTypeB"] = DataType(0) - - if "DataTypeA" in config: - self["DataTypeA"] = DataType(config["DataTypeA"]) - - if "DataTypeB" in config: - self["DataTypeB"] = DataType(config["DataTypeB"]) - - if "DestDataType" in config: - self["DestDataType"] = DataType(config["DestDataType"]) - else: - if "DataType" in config: - self["DestDataType"] = DataType(config["DataType"]) - else: - printExit("NO dest data type or data type specified") - self["DataType"] = DataType(0) - - self["DataTypeE"] = self["DestDataType"] - if "DataTypeE" in config: - self["DataTypeE"] = DataType(config["DataTypeE"]) - - if "ComputeDataType" in config: - self["ComputeDataType"] = DataType(config["ComputeDataType"]) - else: - if "DestDataType" in config: - self["ComputeDataType"] = DataType(config["DestDataType"]) - else: - if "DataType" in config: - self["ComputeDataType"] = DataType(config["DataType"]) - else: - printExit("NO compute data type, or dest data type, or data type specified") - self["DataType"] = DataType(0) - - # Just like DataTypeE is DestDataType by default; DataTypeAmaxD if ComputeDataType by default. - # So far we don't have to set it in config yamls - self["DataTypeAmaxD"] = self["ComputeDataType"] - if "DataTypeAmaxD" in config: - self["DataTypeAmaxD"] = DataType(config["DataTypeAmaxD"]) - - if self["Sparse"]: - self["DataTypeMetadata"] = DataType("I8") - - if "F32XdlMathOp" in config: - self["F32XdlMathOp"] = DataType(config["F32XdlMathOp"]) - else: - self["F32XdlMathOp"] = DataType(0) - - # Modifying ComputeDataType for HHH+HPA: if (HHH+HPA), convert it to HHS_BH by setting ComputeDataType to S. - if self["ComputeDataType"].isHalf() and self["DataType"].isHalf() and self["HighPrecisionAccumulate"]: - printWarning("Inconsistent DataTypes: DataType == f16, DestType == f16, ComputeDataType == f16, but HPA == True (HHH+HPA, no such a type); Converting HHH+HPA to HHS_BH by setting compute data type to f32.") - self["ComputeDataType"] = DataType('s') - - # Modifying ComputeDataType for BBB+HPA: if (BBB+HPA), convert it to BBS_BH by setting ComputeDataType to S. - if self["ComputeDataType"].isBFloat16() and self["DataType"].isBFloat16() and self["HighPrecisionAccumulate"]: - printWarning("Inconsistent DataTypes: DataType == bf16, DestType == bf16, ComputeDataType == bf16, but HPA == True (BBB+HPA, no such a type); Converting BBB+HPA to BBS_BH by setting compute data type to f32.") - self["ComputeDataType"] = DataType('s') - - # Modifying ComputeDataType for I8I8I_BH: if (I8I8I8+HPA), convert it to I8I8I_BH by setting ComputeDataType to i. - if self["ComputeDataType"].isInt8() and DataType(config["DataType"]).isInt8() and self["HighPrecisionAccumulate"]: - print2("DataType == i8 and HPA == True; setting compute data type to int32") - self["ComputeDataType"] = DataType('i') - - if self["OperationType"] == "GEMM": - self.checkIfSupportedGEMMType() - self.initGEMM() - else: - printExit("Unsupported OperationType = %s" % self["OperationType"]) - - self.state["AssignedDerivedParameters"] = False - ProblemType.assignDerivedParameters(self.state) - - for tc in ('A', 'B'): - for sc in self["SetConstStride%s"%tc] : - (anchorDim, stride) = sc[:2] - if anchorDim not in self.state["IndexAssignments%s"%tc]: - printExit("SetConstStride%s=%s anchorDim=%u is not in IndexAssignments%s"%(tc, sc, anchorDim, tc)) - - # Bias - # If compute data type is not equal to dest data type, tensile will run conversion kernel. - # In this case we don't need to apply bias in beta only kernel. - if "UseBias" in config: - if self["ComputeDataType"] != self["DestDataType"]: - self["BetaOnlyUseBias"] = False - else: - self["BetaOnlyUseBias"] = True if self["UseBias"] > 0 else False - if "BiasDataTypeList" in config: - self["BiasDataTypeList"] = [DataType(btype) for btype in config["BiasDataTypeList"]] - self["BiasDataTypeList"].sort() # Make name unique - else: - self["BiasDataTypeList"] = getBiasDataTypeListDefault(self) - else: - self["BetaOnlyUseBias"] = False - self["BiasDataTypeList"] = [] - - # Activation - # Currently, ActivationType supports only 'all' and 'hipblaslt_all', and is active only when the Activation configuration is set to True. - # Otherwise, ActivationType will be set to 'none'. - if "Activation" in config: - typeStr = config.get("ActivationType", 'none') - if typeStr not in ['all', 'hipblaslt_all']: - typeStr = 'none' - else: - typeStr = 'none' - self["ActivationType"] = ActivationType(typeStr) - if "ActivationComputeDataType" in config: - self["ActivationComputeDataType"] = DataType(config["ActivationComputeDataType"]) - else: - self["ActivationComputeDataType"] = self["ComputeDataType"] - - if self["ActivationType"] != 'none': - # This is a dummy guard in case we currently don't have a converter to convert data from compute type to activation compute type - if self["ActivationComputeDataType"] not in [self["ComputeDataType"], self["DestDataType"]]: - printWarning("TensileLite currently only supports ActivationComputeDataType (%s) = ComputeDataType (%s) or DestDataType (%s). \ - ActivationComputeDataType will be set to ComputeDataType automatically."%(self["ActivationComputeDataType"].toChar(), \ - self["ComputeDataType"], \ - self["DestDataType"])) - self["ActivationComputeDataType"] = self["ComputeDataType"] - if (self["ActivationComputeDataType"].numRegisters() != self["ComputeDataType"].numRegisters()) and \ - (self["DataType"].numRegisters() < self["DestDataType"].numRegisters()): - printWarning("TensileLite only supports ActivationComputeDataType = ComputeDataType if DestDataType > DataType. \ - ActivationComputeDataType will be set to ComputeDataType automatically.") - self["ActivationComputeDataType"] = self["ComputeDataType"] - - if "UseE" in config: - if config["UseE"]: - if self["ActivationType"] == 'none': - printWarning("Use E is disabled cause Activation is set to False.") - self["UseE"] = False - else: - self["UseE"] = config["UseE"] - else: - self["UseE"] = config["UseE"] - - if "Gradient" in config: - if config["Gradient"]: - if (not self["UseBias"]) and self["ActivationType"] == 'none': - printWarning("Gradient is disabled cause bias and activation are both disabled.") - self["Gradient"] = False - if self["ActivationType"] != 'none' and self["UseE"] == False: - printWarning("Use E is enabled cause Activation is enabled.") - self["UseE"] = True - elif self["ActivationType"] != 'none' and self["UseE"] == False: - printWarning("Use E is disabled cause Activation is disabled.") - self["UseE"] = False - # if self["UseScaleAlphaVec"]: - # printWarning("Use scaleAlphaVec is disabled cause Gradient is enabled.") - # self["UseScaleAlphaVec"] = False - self["Gradient"] = config["Gradient"] - - # Need gradient info - biasSrcList = ["A", "B", "D"] - if "BiasSrc" in config: - if not self["Gradient"] and config["BiasSrc"] != "D": - printWarning("BiasSrc is set to D cause Gradient is disabled.") - self["BiasSrc"] = "D" - elif self["Gradient"]: - # # Currently only supports D :) - # if config["BiasSrc"] != "D": - # printExit("BiasSrc currently only supports D.") - if config["BiasSrc"] not in biasSrcList: - printExit("BiasSrc only supports A, B, D.") - - if "ActivationNoGuard" in config: - self["ActivationNoGuard"] = config["ActivationNoGuard"] - if self["ActivationNoGuard"]: - if self["ActivationType"] == 'none': - printWarning("ActivationNoGuard is set to False cause Acivation is off.") - self["ActivationNoGuard"] = False - if (not self["Gradient"]): - printWarning("ActivationNoGuard is set to False cause Gradient is off.") - self["ActivationNoGuard"] = False - - ################################################################################ - # Function checkIfSupportedGEMMType: - # Assures 3 data-types are valid, supported and well-assigned - # See the discussion on Common.py for validGEMMTypes - ################################################################################ - def checkIfSupportedGEMMType(self): - inType = self["DataType"] - outType = self["DestDataType"] - computeType = self["ComputeDataType"] - - gemmType = ( inType.toChar(), outType.toChar(), computeType.toChar() ) - if gemmType not in validGEMMTypes: - printExit("This typed-GEMM (Ti, To, Tc) = (%s, %s, %s) is not supported yet."%(gemmType[0],gemmType[1],gemmType[2])) - - ######################################## - def initGEMM(self): - sumIdx = 3 if self["Batched"] else 2 - self["IndexAssignmentsA"] = [0, sumIdx] # N - self["IndexAssignmentsB"] = [sumIdx, 1] # N - if self.state["Sparse"] == 2: - self["IndexAssignmentsMetadata"] = [sumIdx, 1] # N (ref B) - else: - self["IndexAssignmentsMetadata"] = [sumIdx, 0] # T (ref A) - if self["TransposeA"]: - self["IndexAssignmentsA"] = [sumIdx, 0] # T - if self["TransposeB"]: - self["IndexAssignmentsB"] = [1, sumIdx] # T - if self["Batched"]: - self["IndexAssignmentsA"].append(2) - self["IndexAssignmentsB"].append(2) - self["IndexAssignmentsMetadata"].append(2) - self["NumIndicesC"] = 3 - else: - self["NumIndicesC"] = 2 - - self["NumIndicesLD"] = 4 - self["IndexAssignmentsLD"][0] = self["NumIndicesC"] + 1 - for i in range(1, len(self["IndexAssignmentsLD"])): - self["IndexAssignmentsLD"][i] = self["IndexAssignmentsLD"][i-1] + 1 - - ######################################## - def isGEMM(self): - return self.operationType == 0 - - ######################################## - # determine d0, d1, dU - @staticmethod - def assignDerivedParameters(state): - if "AssignedDerivedParameters" in state: - if state["AssignedDerivedParameters"]: - return - state["AssignedDerivedParameters"] = False - - state["TotalIndices"] = max(max(state["IndexAssignmentsA"])+1, \ - max(state["IndexAssignmentsB"])+1) - - # determine num free, batch - state["IndicesFree"] = [] - state["IndicesBatch"] = [] - state["IndicesSummation"] = [] - - for i in range(0, state["NumIndicesC"]): - inA = i in state["IndexAssignmentsA"] - inB = i in state["IndexAssignmentsB"] - if inA and inB: - state["IndicesBatch"].append(i) - - elif inA or inB: - state["IndicesFree"].append(i) - else: - printExit("invalid index %u (inC but not (inA or inB))" % i) - - # determine num summation - for i in range(state["NumIndicesC"], state["TotalIndices"]): - inA = i in state["IndexAssignmentsA"] - inB = i in state["IndexAssignmentsB"] - if inA and inB: - state["IndicesSummation"].append(i) - else: - printExit("invalid index %u (expected summation but not (inA and inB))" % i) - # print index assignments - if globalParameters["PrintIndexAssignments"]: - print("IndicesFree: %s" % state["IndicesFree"]) - print("IndicesBatch: %s" % state["IndicesBatch"]) - print("IndicesSum: %s" % state["IndicesSummation"]) - print("IndexAssignmentsA: %s" % state["IndexAssignmentsA"]) - print("IndexAssignmentsB: %s" % state["IndexAssignmentsB"]) - print("NumIndicesC: %s" % state["NumIndicesC"]) - - for k in ('IndexAssignmentsA','IndexAssignmentsB'): - if len(state[k]) != len(set(state[k])): - printExit("duplicate index in %s=%s"% (k,state[k])) - - state["NumIndicesFree"] = len(state["IndicesFree"]) - state["NumIndicesBatch"] = len(state["IndicesBatch"]) - state["NumIndicesSummation"] = len(state["IndicesSummation"]) - if not state["AllowNoFreeDims"] and state["NumIndicesFree"] < 2 : - printExit("Tensile requires >= 2 free indices or set AllowNoFreeDims; FreeIndices=%s."% state["IndicesFree"]) - - # by default, unroll index will be the last/inner summation index - state["IndexUnroll"] = state["IndicesSummation"][len(state["IndicesSummation"])-1] - for i in range(0, len(state["IndexAssignmentsA"])): - if state["IndexAssignmentsA"][i] == state["IndexUnroll"]: - state["IndexUnrollA"] = i - break - for i in range(0, len(state["IndexAssignmentsB"])): - if state["IndexAssignmentsB"][i] == state["IndexUnroll"]: - state["IndexUnrollB"] = i - break - for i in range(0, len(state["IndexAssignmentsMetadata"])): - if state["IndexAssignmentsMetadata"][i] == state["IndexUnroll"]: - state["IndexUnrollM"] = i - break - #print2("IndexUnrollA: %u" % state["IndexUnrollA"]) - #print2("IndexUnrollB: %u" % state["IndexUnrollB"]) - - # assign d0, d1 - if state["AllowNoFreeDims"]: - dimList = state["IndicesFree"] + state["IndicesBatch"] - else: - dimList = state["IndicesFree"] - state["Index01A"] = [i for i in state["IndexAssignmentsA"] if i in dimList][0] - state["Index01B"] = [i for i in state["IndexAssignmentsB"] if i in dimList][0] - #print2("Index01A: %u" % state["Index01A"]) - #print2("Index01B: %u" % state["Index01B"]) - # Store code is optimized for 0 as the fastest-moving in memory - # whichever has lower stride in C (lower value), is 0, other is 1 - if state["Index01A"] < state["Index01B"]: - state["Index0"] = state["Index01A"] - state["Index1"] = state["Index01B"] - state["Tensor0"] = 0 - state["Tensor1"] = 1 - state["TileA"] = 0 - state["TileB"] = 1 - else: - state["Index0"] = state["Index01B"] - state["Index1"] = state["Index01A"] - state["Tensor0"] = 1 - state["Tensor1"] = 0 - state["TileA"] = 1 - state["TileB"] = 0 - - # generalize transpose - strideIdxA = state["IndexAssignmentsA"].index(state["Index01A"]) - strideIdxB = state["IndexAssignmentsB"].index(state["Index01B"]) - unrollIdxA = state["IndexAssignmentsA"].index(state["IndexUnroll"]) - unrollIdxB = state["IndexAssignmentsB"].index(state["IndexUnroll"]) - state["TLUA"] = strideIdxA < unrollIdxA - state["TLUB"] = strideIdxB < unrollIdxB - #state["TLUB"] = True # hack - - if globalParameters["PrintIndexAssignments"]: - print("TLUA: %s (stridePosA(%d) 1 : - name += "_FD%s"%("N" if factorDim == 2 else "MN") - - if self["UseE"]: - if self["Gradient"]: - name += "_Grad%s"%self["DataTypeE"].toChar() - else: - name += "_Aux%s"%self["DataTypeE"].toChar() # Not showing aux types - if self["OutputAmaxD"]: - name += "_AmaxD" - if self["Sparse"]: - if self["Sparse"] == 2: - name += "_SPB" - else: - name += "_SPA" - - # precision and other - # name += "_SB" if self["StridedBatched"] else "_GB" - if self["GroupedGemm"]: - name += "_GG" - else: - name += "" if self["StridedBatched"] else "_GB" # legacy - - # Activation Naming - if self["ActivationType"] != 'none': - if self["ActivationType"] == 'all': - name += "_A" - elif self["ActivationType"] == 'hipblaslt_all': - name += "_HA" - else: - name += "_%s"%str(self["ActivationType"]).upper() - name += self["ActivationComputeDataType"].toChar() - if self["ActivationNoGuard"]: name += "NG" - - if self["UseScaleAB"] == "Scalar": name += "_SAB" - elif self["UseScaleAB"] == "Vector": name += "_SABV" - if self["UseScaleCD"]: name += "_SCD" - if self["UseScaleAlphaVec"]: name += "_SAV" - - if self["SupportUserArgs"]: name += "_UserArgs" - - return name - - def keys(self): - return list(self.state.keys()) - def __len__(self): - return len(self.state) - def __iter__(self): - return iter(self.state) - def __getitem__(self, key): - return self.state[key] - def __setitem__(self, key, value): - self.state[key] = value - def __repr__(self): - return self.__str__() - def getAttributes(self): - return self.state - def __hash__(self): - return hash(str(self)) - def __eq__(self, other): - return isinstance(other, ProblemType) and self.getAttributes() == other.getAttributes() - def __ne__(self, other): - result = self.__eq__(other) - if result is NotImplemented: - return result - return not result - - def get(self, key, default=None): - try: - return self.state[key] - except: - return default - - - -################################################################################ -# ProblemSizeRange -################################################################################ -class ProblemSizeRange: - - ######################################## - def __init__(self, problemType, config): - self.totalIndices = 1+max(problemType["IndexAssignmentsA"]) + problemType["NumIndicesLD"] - if len(config) < self.totalIndices: - for i in range(len(config), self.totalIndices): - if i < self.totalIndices - problemType["NumIndicesLD"]: - config.append(0) - else: - config.append([0]) - - self.indexMax = [] - self.indexIsSized = [] - self.indicesSized = [] - self.indicesMapped = [] - for i in range(0, self.totalIndices): - dim = deepcopy(config[i]) - if isinstance(dim, list): - if len(dim) == 1: - self.indicesSized.append([dim[0], 1, 0, dim[0]]) - elif len(dim) == 2: - self.indicesSized.append([dim[0], dim[0], 0, dim[1]]) - elif len(dim) == 3: - self.indicesSized.append([dim[0], dim[1], 0, dim[2]]) - elif len(dim) == 4: - self.indicesSized.append([dim[0], dim[1], dim[2], dim[3]]) - else: - printExit("dimension[%u] config (%s) has %u descriptors rather than 1-4." - % ( i, dim, len(dim) )) - self.indexIsSized.append(True) - self.indexMax.append(self.indicesSized[len(self.indicesSized)-1][3]) - - elif isinstance(dim, int): - self.indicesMapped.append(dim) - self.indexIsSized.append(False) - self.indexMax.append(self.indicesSized[self.indicesMapped[ \ - len(self.indicesMapped)-1]][3]) - - # max num elements in each tensor - self.maxNumElements = [ 1, 1, 1 ] - for i in range(0, problemType["NumIndicesC"]): - self.maxNumElements[0] *= self.indexMax[i] - for i in problemType["IndexAssignmentsA"]: - self.maxNumElements[1] *= self.indexMax[i] - for i in problemType["IndexAssignmentsB"]: - self.maxNumElements[2] *= self.indexMax[i] - - self.totalProblemSizes = 1 - self.numProblemSizes = [] # per index - self.problemSizeToIndex = [] - self.problemIndexToSize = [] - sizedIdx = 0 - for i in range(0, len(self.indexIsSized)): - self.problemSizeToIndex.append({}) - self.problemIndexToSize.append({}) - if self.indexIsSized[i]: - self.numProblemSizes.append(0) - index = self.indicesSized[sizedIdx] - sizedIdx += 1 - currentSize = index[0] - currentIncrement = index[1] - while currentSize <= index[3]: - currentSize += currentIncrement - currentIncrement += index[2] - self.numProblemSizes[i] += 1 - else: - self.numProblemSizes.append(1) - self.totalProblemSizes *= self.numProblemSizes[i] - - ######################################## - # enumerate problem sizes - currentSizedIndexSizes = [] - currentSizedIndexIncrements = [] - for i in range(0, len(self.indicesSized)): - currentSizedIndexSizes.append(self.indicesSized[i][0]) - currentSizedIndexIncrements.append(self.indicesSized[i][1]) - - # iterate over all problem sizes - self.problemSizes = [] - moreProblemSizes = True - problemIdx = 0 - problemSize = [0]*self.totalIndices - while moreProblemSizes: - #/ convert current sized and mapped indices to full sizes - currentSizedIdx = 0 - currentMappedIdx = 0 - for i in range(0, self.totalIndices): - if self.indexIsSized[i]: - problemSize[i] = currentSizedIndexSizes[currentSizedIdx] - currentSizedIdx+=1 - else: - problemSize[i] = problemSize[self.indicesMapped[currentMappedIdx]] - currentMappedIdx+=1 - self.problemSizes.append(tuple(problemSize)) - - #/ increment sizes for next benchmark - currentSizedIndexSizes[0] += currentSizedIndexIncrements[0] - currentSizedIndexIncrements[0] += self.indicesSized[0][2] - for i in range(1, len(self.indicesSized)+1): - # if prior index past max, reset to min and increment next index - if currentSizedIndexSizes[i-1] > self.indicesSized[i-1][3]: - #/ reset prior index - currentSizedIndexSizes[i-1] = self.indicesSized[i-1][0] - currentSizedIndexIncrements[i-1] = self.indicesSized[i-1][1] - # increment next index - if i >= len(self.indicesSized): - moreProblemSizes = False - else: - currentSizedIndexSizes[i] += currentSizedIndexIncrements[i] - currentSizedIndexIncrements[i] += self.indicesSized[i][2] - - problemIdx+=1 - - ######################################## - # YAML format - def __str__(self): - state = "[ " - sizedIdx = 0 - mappedIdx = 0 - for i in range(0, len(self.indexIsSized)): - if self.indexIsSized[i]: - indices = self.indicesSized[sizedIdx] - state += "[ %u, %u, %u, %u ]" \ - % (indices[0], indices[1], indices[2], indices[3]) - sizedIdx += 1 - else: - indices = self.indicesSized[self.indicesMapped[mappedIdx]] - state += str(self.indicesMapped[mappedIdx]) - mappedIdx += 1 - if i < len(self.indexIsSized)-1: - state += ", " - state += " ]" - return state - -class Problem: - """ Problem sizes, strides, padding and other info""" - def __init__(self, sizes=None, stridesA=None, stridesB=None, stridesC=None, stridesD=None, count=None): - self.sizes = tuple(sizes) if sizes else None - self.stridesA = tuple(stridesA) if stridesA else None - self.stridesB = tuple(stridesB) if stridesB else None - self.stridesC = tuple(stridesC) if stridesC else None - self.stridesD = tuple(stridesD) if stridesD else None - - self.count = count - - def __str__(self): - rv= "{ sizes:" + str(list(self.sizes)) - if self.stridesA: - rv += ", stridesA:" + str(list(self.stridesA)) - if self.stridesB: - rv += ", stridesB:" + str(list(self.stridesB)) - if self.stridesC: - rv += ", stridesC:" + str(list(self.stridesC)) - if self.stridesD: - rv += ", stridesD:" + str(list(self.stridesD)) - rv += " }" - return rv - -class ExactList(Problem): - def __init__(self, e, problemType): - if len(e) == problemType["TotalIndices"]: - if -1 in e: - printExit("ExactSize %s contains -1" % (e)) - if problemType["OperationType"] == "GEMM": - e += [-1, -1, -1, -1] - e = ExactList.convertLeadingDims(problemType, tuple(e)) - sizes=e - - elif len(e) == (problemType["TotalIndices"] + problemType["NumIndicesLD"]): - sizes = ExactList.convertLeadingDims(problemType, tuple(e)) - else: - printExit("ExactSize %s doesn't match indices of ProblemType %s, totalIndices=%d, len e=%d, NumIndicesLD = %d" \ - % (e, problemType, problemType["TotalIndices"], len(e), problemType["NumIndicesLD"]) ) - - # TODO- pass strides here, remove calls to convertLeadingDims - Problem.__init__(self, sizes=sizes) - - def __str__(self): - return str(list(self.sizes)) - - @staticmethod - def convertLeadingDims(problemType, problemSize, stridesA = None, stridesB = None, stridesC = None, stridesD = None): - # FIXME-problem: refactor to eliminate max, pass strides in strideB parm rather than hacked - # onto the end of the sizes list - predStridesD = stridesD is not None and stridesD[1] != -1 - predStridesC = stridesC is not None and stridesC[1] != -1 - predStridesA = stridesA is not None and stridesA[1] != -1 - predStridesB = stridesB is not None and stridesB[1] != -1 - return problemSize[:problemType["NumIndicesC"]+1] + \ - (max(problemSize[0], problemSize[problemType["IndexAssignmentsLD"][0]]) if not predStridesD else stridesD[1], ) + \ - (max(problemSize[0], problemSize[problemType["IndexAssignmentsLD"][1]]) if not predStridesC else stridesC[1], ) + \ - (max(problemSize[problemType["IndexAssignmentsLD"][2]], - problemSize[problemType["IndexAssignmentsA"][0]]) if not predStridesA else stridesA[1], ) + \ - (max(problemSize[problemType["IndexAssignmentsLD"][3]], - problemSize[problemType["IndexAssignmentsB"][0]]) if not predStridesB else stridesB[1], ) - - -class ExactDict(Problem): - AllowedFields = [ 'count', 'sizes', 'stridesA', 'stridesB', 'stridesC', 'stridesD' ] - - def __init__(self, e, problemType): - Problem.__init__(self) - - for f in e: - if f in ExactDict.AllowedFields: - setattr(self, f, e[f]) - else: - raise RuntimeError ("specified field '%s' is not a valid Exact dict field"%f) - - if problemType: - if "OperationType" in problemType and problemType["OperationType"] == "GEMM": - sizesTuple = tuple(self.sizes + [-1, -1, -1, -1]) - self.sizes = ExactList.convertLeadingDims(problemType, sizesTuple, self.stridesA, self.stridesB, self.stridesC, self.stridesD) - - if problemType: - if "OperationType" in problemType and problemType["OperationType"] == "GEMM": - if len(self.sizes) != (problemType["TotalIndices"] + problemType["NumIndicesLD"]): - # FIXME-ExactDict size descriptor still (but preferrably not so) uses 8-tuple for GEMM problems - raise RuntimeError ("specified size=%s does not have enough indices for problem (expected %d, got %d)" \ - % (self.sizes, problemType["TotalIndices"]+problemType["NumIndicesLD"], len(self.sizes))) - elif len(self.sizes) != problemType["TotalIndices"]: - raise RuntimeError ("specified size=%s does not have enough indices for problem (expected %d, got %d)" \ - % (self.sizes, problemType["TotalIndices"], len(self.sizes))) - - -################################################################################ -# ProblemSizes -################################################################################ -""" -Adapter class for class `ProblemSizes`. It satisfies the implicit usage requirement -of ClientWriter.writeClientConfig() by converting ExactLogic to list of `Problem` objects -""" -class ProblemSizesMock: - def __init__(self, exactLogic): - self.problems = [Problem(problem) for problem, solution in exactLogic] - -class ProblemSizesMockDummy: - def __init__(self): - self.problems = [Problem(sizes=[128, 128, 1, 512])] - -class ProblemSizes: - - ######################################## - def __init__(self, problemType, config): - self.problemType = problemType - self.ranges = [] - self.exacts = [] - self.minStrides = None - if config: - for dictionary in config: - for sizeTypeKey in dictionary: - #print ("PROBLEM parsed:", sizeTypeKey, dictionary[sizeTypeKey]) - if sizeTypeKey == "Range": - psr = ProblemSizeRange(problemType, dictionary[sizeTypeKey]) - self.ranges.append( psr ) - elif sizeTypeKey == "Exact": - e= dictionary[sizeTypeKey] - if isinstance(e,list): - self.exacts.append(ExactList(e, problemType)) - elif isinstance(e,dict): - self.exacts.append(ExactDict(e, problemType)) - else: - printExit("Unsupported Exact type==%s"%type(e)) - elif sizeTypeKey == "MinStride": - e = dictionary[sizeTypeKey] - if len(e) != problemType["TotalIndices"]: - printExit("MinStride %s doesn't match indices of ProblemType %s" \ - % (e, problemType) ) - if self.minStrides: - printExit("Only one MinStride command is allowed in a ProblemsSizes definition. Previous minStrides:%s, New minstride:%s" \ - % (self.minStrides, e) ) - - self.minStrides=(tuple(e)) - else: - printExit("ProblemSize Type %s not supported"%sizeTypeKey) - - if not self.minStrides: - # set harmless default mins of 0 - self.minStrides = ([0]* problemType["TotalIndices"]) - - # not the ideal spot, but convert leading dims that are below the minimum size - if problemType["OperationType"] == "GEMM": - for i in range(0, len(self.ranges)): - self.ranges[i].problemSizes[:] = \ - [ExactList.convertLeadingDims(self.problemType, problemSize) for problemSize in self.ranges[i].problemSizes] - - self.problems = OrderedDict() - for sizeRange in self.ranges: - for rangeSize in sizeRange.problemSizes: - self.problems.update({Problem(rangeSize) : 1}) - for e in self.exacts: - self.problems.update({e : 1}) - self.problems = list(self.problems.keys()) - self.totalProblemSizes = len(self.problems) - - # max sizes - self.maxD = 0 - self.maxC = 0 - self.maxA = 0 - self.maxB = 0 - for problem in self.problems: - problemSize = problem.sizes # FIXME-problem. This should use problem.strides* - - sizeLdd = problemSize[self.problemType["IndexAssignmentsLD"][0]] if problemType["OperationType"] == "GEMM" else problemSize[0] - sizeD = max(self.minStrides[0], sizeLdd) - for i in range(1, problemType["NumIndicesC"]): - sizeD *= max(self.minStrides[i], problemSize[i]) - - sizeLdc = problemSize[self.problemType["IndexAssignmentsLD"][1]] if problemType["OperationType"] == "GEMM" else problemSize[0] - sizeC = max(self.minStrides[0], sizeLdc) - for i in range(1, problemType["NumIndicesC"]): - sizeC *= max(self.minStrides[i], problemSize[i]) - - sizeLda = problemSize[self.problemType["IndexAssignmentsLD"][2]] \ - if problemType["OperationType"] == "GEMM" \ - else problemSize[self.problemType["IndexAssignmentsA"][0]] - sizeA = max(self.minStrides[self.problemType["IndexAssignmentsA"][0]], sizeLda) - for i in self.problemType["IndexAssignmentsA"][1:]: - sizeA *= max(self.minStrides[i], problemSize[i]) - - sizeLdb = problemSize[self.problemType["IndexAssignmentsLD"][3]] \ - if problemType["OperationType"] == "GEMM" \ - else problemSize[self.problemType["IndexAssignmentsB"][0]] - sizeB = max(self.minStrides[self.problemType["IndexAssignmentsB"][0]], sizeLdb) - for i in self.problemType["IndexAssignmentsB"][1:]: - sizeB *= max(self.minStrides[i], problemSize[i]) - - self.maxD = max(self.maxD, sizeD) - self.maxC = max(self.maxC, sizeC) - self.maxA = max(self.maxA, sizeA) - self.maxB = max(self.maxB, sizeB) - - def __str__(self): - s = "ProblemSizes\n" - for sizeRange in self.ranges: - s += " %s" % sizeRange - return s ################################################################################ # Factor Type ################################################################################ - class FactorDimArgs: ######################################## @@ -957,23 +95,6 @@ def __str__(self): s = "FactorDimArgs\n" return s -################################################################################ -# Bias Type -################################################################################ - -def getBiasDataTypeListDefault(problem: ProblemType) -> List[DataType]: - bList = [] - for d in ["DataType", "ComputeDataType", "DestDataType"]: - dtype = DataType(problem[d]) - # filter out int8, because it is not supported by bias datatype - # TODO - if not dtype.isInt8(): - bList.append(dtype) - - biasDataTypeList = list(set(bList)) - biasDataTypeList.sort() # Make name unique - return biasDataTypeList - class BiasTypeArgs: ######################################## @@ -1052,19 +173,33 @@ def isExtractableIndex(ks, index, tc='x'): ################################################################################ class Solution(collections.abc.Mapping): - ######################################## - def __init__(self, config, cxxCompiler: str, srcName: str = ""): + ######################################## # need to be sure PSRR is passing to all fxns + def __init__( + self, + config, + splitGSU: bool, + printSolutionRejectionReason: bool, + printIndexAssignmentInfo: bool, + depthUConfig: DepthUConfig, + assembler: Assembler, + isaInfoMap: Dict[IsaVersion, IsaInfo], + srcName: str = "" + ): + self._name = None - self.cxxCompiler = cxxCompiler + self.assembler = assembler + self.isaInfoMap = isaInfoMap self.srcName = srcName + self.splitGSU = splitGSU config = config + targetIsas = list(isaInfoMap.keys()) self._state = {} # problem type if "ProblemType" in config: - self["ProblemType"] = ProblemType(config["ProblemType"]) + self["ProblemType"] = ProblemType(config["ProblemType"], printIndexAssignmentInfo) else: - self["ProblemType"] = ProblemType(defaultProblemType) + self["ProblemType"] = ProblemType.FromDefaultConfig(printIndexAssignmentInfo) if "InternalSupportParams" in config: self["InternalSupportParams"] = {} @@ -1073,30 +208,27 @@ def __init__(self, config, cxxCompiler: str, srcName: str = ""): else: self["InternalSupportParams"] = defaultInternalSupportParams - - # assign parameters with defaults + # Assign solution state from config, filling missing from the defaultSolution for key in defaultSolution: assignParameterWithDefault(self._state, key, config, defaultSolution) + if 'ISA' not in self._state: if 'ISA' in config: - if not globalParameters["AsmCaps"][tuple(config['ISA'])]["SupportedISA"]: - defaultIsa = [9,0,0] - print("warning: ISA:", config['ISA'], " is not supported; overriding with ", defaultIsa) - self._state['ISA'] = defaultIsa - else: - self._state['ISA'] = config['ISA'] + # The ISA is expected to be defined when calling from TensileCreateLibrary + isa = config['ISA'] + isa = IsaVersion(isa[0], isa[1], isa[2]) + assert self.isaInfoMap[isa].asmCaps["SupportedISA"] + self._state['ISA'] = IsaVersion(isa[0], isa[1], isa[2]) else: - # Assembly by default - self._state['ISA'] = list(globalParameters["CurrentISA"]) - if 'KernelLanguage' in config: - if config['KernelLanguage'] != 'Assembly': - self._state['ISA'] = [0,0,0] + # When calling from Tensile, the ISA is typically not defined. + printWarning(f"ISA not set on config using {targetIsas[0]}.") + self._state['ISA'] = targetIsas[0] if "CodeObjectVersion" not in self._state: if "CodeObjectVersion" in config: self._state["CodeObjectVersion"] = str(config["CodeObjectVersion"]) else: - self._state["CodeObjectVersion"] = str(globalParameters["CodeObjectVersion"]) + self._state["CodeObjectVersion"] = self.assembler.code_object_version # assign parameters without defaults for key in config: if (key != "ProblemType" or key != "InternalSupportParams") and key not in self._state: @@ -1107,9 +239,18 @@ def __init__(self, config, cxxCompiler: str, srcName: str = ""): self["AssignedProblemIndependentDerivedParameters"] = False if "AssignedDerivedParameters" not in self._state: self["AssignedDerivedParameters"] = False - Solution.assignDerivedParameters(self._state) - self._name = config["CustomKernelName"] if isCustomKernelConfig(config) else None - self.initHelperKernelObjects() + Solution.assignDerivedParameters( + self._state, + splitGSU, + printSolutionRejectionReason, + printIndexAssignmentInfo, + isaInfoMap, + assembler.rocm_version, + depthUConfig, + ) + self._name = config["CustomKernelName"] if "CustomKernelName" in config and config["CustomKernelName"] else None + + self.initHelperKernelObjects(targetIsas) # these keys are copied from ProblemType to internal that may be overridden InternalKeys = ["UseSgprForGRO","VectorStore"] @@ -1127,11 +268,11 @@ def getKernels(self): ######################################## # create Helper Kernels - def initHelperKernelObjects(self): + def initHelperKernelObjects(self, supportedISA: List[IsaVersion]): self.initBetaOnlyKernelObjects() - self.initConversionKernelObjects() + self.initConversionKernelObjects(supportedISA) self.initActivationEnumHeaderObjects() - self.initActivationFunctionObjects() + self.initActivationFunctionObjects(supportedISA) self.initActivationOnlyKernelObjects() self.initReductionKernelObjects() @@ -1161,7 +302,7 @@ def initBetaOnlyKernelObjects(self): ######################################## # create Conversion Kernels - def initConversionKernelObjects(self): + def initConversionKernelObjects(self, supportedArchs: List[tuple]): self.conversionKernelObjects = [] load_vector_width = [1, 2] if self["ProblemType"]["DataType"].isDouble() else [1, 2, 4] genPGRPostKernels = True @@ -1188,7 +329,7 @@ def initConversionKernelObjects(self): state["UnrollOnly"] = unrollOnly state["_GlobalAccumulation"] = self["_GlobalAccumulation"] state["ActivationFused"] = self["ActivationFused"] - self.conversionKernelObjects.append(KernelWriterConversion(state, vw)) + self.conversionKernelObjects.append(KernelWriterConversion(state, vw, supportedArchs, self.isaInfoMap)) for btype in typeList: state = {} state["ProblemType"] = deepcopy(self["ProblemType"]) @@ -1201,7 +342,7 @@ def initConversionKernelObjects(self): state["UnrollOnly"] = unrollOnly state["_GlobalAccumulation"] = self["_GlobalAccumulation"] state["ActivationFused"] = self["ActivationFused"] - self.conversionKernelObjects.append(KernelWriterConversion(state, vw)) + self.conversionKernelObjects.append(KernelWriterConversion(state, vw, supportedArchs, self.isaInfoMap)) else: state = {} state["ProblemType"] = deepcopy(self["ProblemType"]) @@ -1212,7 +353,7 @@ def initConversionKernelObjects(self): state["UnrollOnly"] = unrollOnly state["_GlobalAccumulation"] = self["_GlobalAccumulation"] state["ActivationFused"] = self["ActivationFused"] - self.conversionKernelObjects.append(KernelWriterConversion(state, vw)) + self.conversionKernelObjects.append(KernelWriterConversion(state, vw, supportedArchs, self.isaInfoMap)) def initActivationEnumHeaderObjects(self): self.activationEnumHeaderObjects = [] @@ -1223,7 +364,7 @@ def initActivationEnumHeaderObjects(self): state["KernelLanguage"] = "Source" self.activationEnumHeaderObjects.append(KernelWriterActivationEnumHeader(state)) - def initActivationFunctionObjects(self): + def initActivationFunctionObjects(self, supportedISA: List[IsaVersion]): self.activationFunctionObjects = [] if self["ProblemType"]["ActivationType"] in ['all', 'hipblaslt_all']: state = {} @@ -1231,7 +372,9 @@ def initActivationFunctionObjects(self): state["ProblemType"]["GroupedGemm"] = False state["KernelLanguage"] = "Source" state["Kernel"] = {"WavefrontSize": self["WavefrontSize"], "ISA": tuple(self["ISA"])} - self.activationFunctionObjects.append(KernelWriterActivationFunction(state, self.cxxCompiler)) + if not isinstance(supportedISA, list): + raise Exception(f"{type(supportedISA)}") + self.activationFunctionObjects.append(KernelWriterActivationFunction(state, str(self.assembler.path), supportedISA)) def initActivationOnlyKernelObjects(self): self.activationOnlyKernelObjects = [] @@ -1277,19 +420,19 @@ def getKernelConversionObjects(self): return self.conversionKernelObjects @staticmethod - def getMIOutputInfo(state): + def getMIOutputInfo(state, isaInfoMap: Dict[str, IsaInfo]): outputVectorWidth = 4 RegsPerOut = 1 isa = tuple(state["ISA"]) - if globalParameters["AsmCaps"][isa]['HasMFMA']: + if isaInfoMap[isa].asmCaps['HasMFMA']: if state["ProblemType"]["DataType"].MIOutputTypeNameAbbrev() == 'f64': outputVectorWidth, RegsPerOut = 1, 2 else: outputVectorWidth, RegsPerOut = 4, 1 - elif globalParameters["AsmCaps"][isa]['HasWMMA_V1']: + elif isaInfoMap[isa].asmCaps['HasWMMA_V1']: outputVectorWidth, RegsPerOut = 1, 1 - elif globalParameters["AsmCaps"][isa]['HasWMMA_V2']: + elif isaInfoMap[isa].asmCaps['HasWMMA_V2']: outputVectorWidth, RegsPerOut = 8, 1 else: print("WARNING: unexpect code flow") @@ -1299,11 +442,7 @@ def getMIOutputInfo(state): ######################################## # assign tile sizes @staticmethod - def assignProblemIndependentDerivedParameters(state): - - if globalParameters["NewClient"] != 2: - print("WARNING: Old client deprecated, NewClient parameter being set to 2.") - globalParameters["NewClient"] = 2 + def assignProblemIndependentDerivedParameters(state, printRejectionReason: bool, isaInfoMap: Dict[str, IsaInfo]): if "AssignedProblemIndependentDerivedParameters" in state: if state["AssignedProblemIndependentDerivedParameters"]: @@ -1313,12 +452,13 @@ def assignProblemIndependentDerivedParameters(state): state["Valid"] = True if (not state["ProblemType"]["StridedBatched"]) and (not state["ProblemType"]['Batched']): - reject(state, "General Batched GEMM only support Batched Problem") + reject(state, printRejectionReason, "General Batched GEMM only support Batched Problem") if (not state["ProblemType"]["StridedBatched"]) and (state["ProblemType"]["OperationType"] != 'GEMM'): - reject(state, "General Batched GEMM only support GEMM OperationType") + reject(state, printRejectionReason, "General Batched GEMM only support GEMM OperationType") + + ### ---> This is where we previously called matrixInstructionToMIParameters - Solution.MatrixInstructionToMIParameters(state) EnableMatrixInstruction = state["EnableMatrixInstruction"] if "EnableMatrixInstruction" in state else None if EnableMatrixInstruction == None: if ("MIBlock" in state and len(state["MIBlock"]) == 6) \ @@ -1329,7 +469,7 @@ def assignProblemIndependentDerivedParameters(state): and ("ThreadTile" in state and len(state["ThreadTile"]) == 2) : EnableMatrixInstruction = False else: - reject(state, "EnableMatrixInstruction undetermined") + reject(state, printRejectionReason, "EnableMatrixInstruction undetermined") if EnableMatrixInstruction == True: state["MatrixInstM"] = state["MIBlock"][0] @@ -1340,7 +480,7 @@ def assignProblemIndependentDerivedParameters(state): state["MatrixInstBN"] = state["MIBlock"][5] state["LocalSplitU"] = 1 - state["MIOutputVectorWidth"], state["MIRegPerOut"] = Solution.getMIOutputInfo(state) + state["MIOutputVectorWidth"], state["MIRegPerOut"] = Solution.getMIOutputInfo(state, isaInfoMap) if state["MatrixInstM"] == 4: state["ThreadTile0"] = state["MIWaveTile"][0] * state["MIOutputVectorWidth"] @@ -1371,7 +511,7 @@ def assignProblemIndependentDerivedParameters(state): if "SubGroup0" in state and "SubGroup1" in state and "LocalSplitU" in state: state["NumThreads"] = state["SubGroup0"] * state["SubGroup1"] * state["LocalSplitU"] if (state["NumThreads"] % state['WavefrontSize']) != 0: - reject(state, f"size of WorkGroup {state['NumThreads']} should be multiple of WavefrontSize {state['WavefrontSize']}") + reject(state, printRejectionReason, f"size of WorkGroup {state['NumThreads']} should be multiple of WavefrontSize {state['WavefrontSize']}") # macro tile sizes if "SubGroup0" in state and "ThreadTile0" in state: @@ -1381,13 +521,13 @@ def assignProblemIndependentDerivedParameters(state): if "MacroTile" in state: if state["MacroTile0"] != state["MacroTile"][0] \ or state["MacroTile1"] != state["MacroTile"][1]: - reject(state, "MacroTile mismatch") + reject(state, printRejectionReason, "MacroTile mismatch") # tail loop optimization state["tailLoopOptA"] = True state["tailLoopOptB"] = True - if (tuple(state["ISA"]) != (9, 4, 2)) or \ + if (state["ISA"] != IsaVersion(9, 4, 2)) or \ (state["ProblemType"]["Sparse"]): state["tailLoopOptA"] = False state["tailLoopOptB"] = False @@ -1420,12 +560,12 @@ def assignProblemIndependentDerivedParameters(state): # state[GlobalReadVectorWidth*] # state[NumLoads*] # only used in SolutionStructs, with classic alg @staticmethod - def setGlobalReadVectorWidth(state, tc, totalVectors, grvw): + def setGlobalReadVectorWidth(state, tc, totalVectors, grvw, printRejectionReason: bool): validDepthU = True if grvw not in [1,2,4,8,16,32]: validDepthU = False if totalVectors % state["NumThreads"] != 0: - reject(None, "totalVectors%s %u %% NumThreads %u != 0" \ + reject(None, printRejectionReason, "totalVectors%s %u %% NumThreads %u != 0" \ % (tc, totalVectors, state["NumThreads"])) validDepthU = False @@ -1448,7 +588,7 @@ def setGlobalReadVectorWidth(state, tc, totalVectors, grvw): # state[LSCA] # state[LSPA] @staticmethod - def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, totalElementsPerp, depthU): + def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, totalElementsPerp, depthU, printRejectionReason: bool): if state["WaveSeparateGlobalRead%s"%tc]: totalElementsPerp = roundupRatio(totalElementsPerp, state["NumThreads"] // state["WavefrontSize"]) @@ -1477,7 +617,7 @@ def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, tota foundValid = True break if not foundValid: - reject(state, "%s: No NumLoadsCoalesced=1 found"%tc) + reject(state, printRejectionReason, "%s: No NumLoadsCoalesced=1 found"%tc) return False # nlc = -1 @@ -1493,29 +633,29 @@ def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, tota foundValid = True break if not foundValid: - reject(state, "%s: No NumLoadsCoalesced=-1 found"%tc) + reject(state, printRejectionReason, "%s: No NumLoadsCoalesced=-1 found"%tc) return False # nlc = other else: if state["NumLoadsCoalesced%s"%tc] > state["NumLoads%s"%tc]: - reject(state, "%s nlc > numLoads"%tc) + reject(state, printRejectionReason, "%s nlc > numLoads"%tc) return False state["NumLoadsPerpendicular%s"%tc] = state["NumLoads%s"%tc] \ // state["NumLoadsCoalesced%s"%tc] if state["NumLoads%s"%tc] % state["NumLoadsCoalesced%s"%tc] != 0: - reject(state, "%s: numLoads %u %% numLoadsCoalesced %u != 0" \ + reject(state, printRejectionReason, "%s: numLoads %u %% numLoadsCoalesced %u != 0" \ % (tc, state["NumLoads%s"%tc], state["NumLoadsCoalesced%s"%tc])) return False if totalVectorsCoalesced % state["NumLoadsCoalesced%s"%tc] != 0 : - reject(state, "%s: totalVectorsCoalesced %u %% numLoadsPara %u != 0" \ + reject(state, printRejectionReason, "%s: totalVectorsCoalesced %u %% numLoadsPara %u != 0" \ % (tc, totalVectorsCoalesced, state["NumLoadsCoalesced%s"%tc])) return False if totalElementsPerp % state["NumLoadsPerpendicular%s"%tc] != 0: - reject(state, "%s: totalElementsPerp %u %% numLoadsPerp %u != 0" \ + reject(state, printRejectionReason, "%s: totalElementsPerp %u %% numLoadsPerp %u != 0" \ % (tc, totalElementsPerp, state["NumLoadsPerpendicular%s"%tc])) return False @@ -1534,289 +674,32 @@ def setGlobalLoadTileDimClassic(state, tc, numLoads, totalVectorsCoalesced, tota return True - ######################################## - # Sets the Global Read Tile dims (para, perp) - # This information controls which threads read which addresses from global mem) - # Output from this function: - # state[NumLoadsCoalesced*] - # state[NumLoadsPerpendicular*] - # state[LSC*] - # state[LSP*] - # state[GlobalReadVectorWidth] - # - # LSC and LSP define the shape of the PerLoadTile, measured in elements. - # LSC*LSP is the elements loaded by a single instruction across all - # threads in the group. - # LSC is the number of elements loaded in the para(coalesced) dimension - # LSP is the number of elements loaded in the perp(noncoalesced) dimension - # PerLoadTile is always rectangular. - # When BufferLoad=1, the area (LSC*LSP) can be larger than NumThreads. - # In this case, some threads will generate a dummy OOB GRO. - # Related fields: - # LVC = LSC/GRVW (LVCA = LSCA/GLVWA) - # LVP = LSP/GRVW (LVPA = LSPA/GLVWA) - # - # NumLoadsCoalesced and NumLoadsPerpendicular define the number of times the - # PerLoadTile is loaded in each dimension to fetch the LoadTile - # LoadTile = (LSC * NumLoadsCoalesced) * (LSP * NumLoadsPerpendicular). - # For Fractional, the LoadTile can be larger than the MacroTile. Buffer - # loads will clip any OOB references to 0 and will also avoid writing these - # into LDS. - - # Fractional load algorithm: - # - Each load instruction loads one or more (complete) rows of the load tile. - # - Each row is LSC elements wide - # - Rows are complete and do not wrap. This allows a single base GRO VGPR - # to be used for all loads in the tile. - # - Some work-items in the load may not perform useful work. These WI will - # set their GRO to a large OOB number so as to do no harm - # - Some G2L registers space may be unused as well. - # - The 'used' message at the bottom of this routine computes and prints the - # wasted register space. - # - The wasted space is removed when the data is written to LDS- the LWO - # for work-items beyond the valid ones are set to safely write to OOB locations. - - # - In cases where each load is loading multiple rows (multiple lines of lsc - # elements), the last load is allowed to load fewer lines than the others. - # The KernelWriterAssembly will modify the LWO for the last load. This allows - # flexibility in the unroll factors for example. - @staticmethod - def setGlobalLoadTileDimFractional(state, tc, depthU): - - assert(depthU > 0) - dbFract = 0 - - # parDim, perpDim define the LoadTile and are measured in elements - if state["ProblemType"]["TLU%s"%tc]: - parDim = state["MacroTile%s"%tc] - perpDim = depthU - else: - parDim = depthU - perpDim = state["MacroTile%s"%tc] - - if dbFract: - print("\ninfo: %s Fractional MT%u_%u_%u Par=%u Perp=%u WG%02u_%02u_%02u NumThreads=%u GRWV%s=%u" \ - % (tc, state["MacroTile0"], state["MacroTile1"], depthU, \ - parDim, perpDim, \ - state["WorkGroup"][0], state["WorkGroup"][1], state["LocalSplitU"], \ - state["NumThreads"], tc, state["GlobalReadVectorWidth%s"%tc])) - - # Try to find a GRVW which is smaller than the LSC and also does not force - # the LSC to wrap - both of these conditions can be tested with lsc % grvw ==0. - # Each iteration divides GRWV by 2 which provides finer granularity - # and a possible opportunity to handle the lsc - grvw = state["GlobalReadVectorWidth%s"%tc] - minGrvw = 2 if state["ProblemType"]["DataType"].isHalf() and \ - globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"] else 1 - # TODO- check this for int8 and fractional load - # minGrvw = 4 if state["ProblemType"]["DataType"].isInt8() and \ - # globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"] else 1 - bestVw = -1 - while grvw >= minGrvw: - # Per instruction across the entire group: - elementsLoadedPerInst = state["NumThreads"]*grvw - mik = 1 - if (state["DirectToVgpr%s"%tc] and state["ProblemType"]["TLU%s"%tc]): - mik = state["MatrixInstK"] * state["LocalSplitU"] // state["MIInputPerThread"] - elementsLoadedPerInst //= mik - # LSC, LSP - #elements loaded along specified dim with each load - if parDim >= elementsLoadedPerInst: - # entire work-group can work on (part) of the same row - state["LSC%s"%tc] = elementsLoadedPerInst - state["LSP%s"%tc] = mik if state["ProblemType"]["TLU%s"%tc] else state["MatrixInstK"] - state["NumLoadsCoalesced%s"%tc] = roundupRatio(parDim , state["LSC%s"%tc]) - state["NumLoadsPerpendicular%s"%tc] = 1 - else: - # work-group exceeds read dimension so wraps to multiple rows - state["LSC%s"%tc] = parDim - state["LSP%s"%tc] = min(perpDim, elementsLoadedPerInst // parDim) - state["NumLoadsCoalesced%s"%tc] = 1 - state["NumLoadsPerpendicular%s"%tc] = roundupRatio(perpDim , state["LSP%s"%tc]) - - # Vector loads can't wrap to next P dim, so LSC must be divisible by vector elements; - if dbFract: - print(" lsc search : lsc(%u) %% grvw(%u) = %u (?0)" % (state["LSC%s"%tc], grvw, state["LSC%s"%tc] % grvw)) - if state["LSC%s"%tc] % grvw == 0: - bestVw = grvw - # Try to shrink GRVW if possible while keeping same LSC and LSP: - # For example, avoid cases where we use a GRVW=4 with many empty addresses - # when a GRVW=1 will do instead. - validElementsLoadedPerInst = state["LSC%s"%tc] * state["LSP%s"%tc] - grvw //= 2 - while grvw >= minGrvw: - elementsLoadedPerInst = state["NumThreads"]*grvw//mik - if elementsLoadedPerInst < validElementsLoadedPerInst: - break # Went too far, not enough load elements at this VW - if state["LSC%s"%tc] % grvw == 0: - if dbFract: - print(" stepdown success (valid)elementsLoadedPerInst=", validElementsLoadedPerInst, "/", elementsLoadedPerInst, "grvw=", grvw, "lsc=", state["LSC%s"%tc]) - bestVw = grvw - grvw //= 2 - break - - # TODO - could have this generate dwordx3 loads in addition, step down by 1 instead of div2 - # Would need to change asm code gen to generate x3 - grvw //= 2 - # end-- while loop - - if bestVw == -1: - if dbFract: - print ("reject fractional - no acceptable tile dim? GlobalReadVectorWidth%s"%tc, \ - state["GlobalReadVectorWidth%s"%tc]) - return False # could not find a solution, perhaps only possible for half ? - - state["GlobalReadVectorWidth%s"%tc] = bestVw - if bestVw != state["GlobalReadVectorWidth%s"%tc]: - if dbFract: - print(" reducing GlobalReadVectorWidth%s from %u to %u" \ - % (tc, state["GlobalReadVectorWidth%s"%tc], bestVw)) - - # How many loads per threads in each dimension. - # threads which are outside the global read tile bounds will be clipped - # in the assembly code generator. - # Multiply the LSC*GRVW - state["NumLoadsCoalesced%s"%tc] = roundupRatio(parDim, state["LSC%s"%tc]) - state["NumLoadsPerpendicular%s"%tc] = roundupRatio(perpDim , state["LSP%s"%tc]) - - nlc = state["NumLoadsCoalesced%s"%tc] - nlp = state["NumLoadsPerpendicular%s"%tc] - - # LoadTile must at least cover the MacroTile: - assert(nlc*state["LSC%s"%tc] >= parDim) - assert(nlp*state["LSP%s"%tc] >= perpDim) - - perpOverhang = perpDim % state["LSP%s"%tc] - state["fractionalPerpOverhang%s"%tc] = perpOverhang - if dbFract: - # how many threads compute Global Read Offsets (GRO) that are not used - print(" PerLoadTile=%ux%u elements Loads/WI=%ux%u LoadTile/WI=%ux%u (MT=%ux%u), %u/%u = %.1f%% WI GRO used %s" \ - % (state["LSC%s"%tc], state["LSP%s"%tc], \ - nlc, nlp, \ - nlc*state["LSC%s"%tc], nlp*state["LSP%s"%tc], \ - parDim, perpDim, \ - parDim*perpDim, \ - nlc*nlp*state["NumThreads"]*state["GlobalReadVectorWidth%s"%tc], \ - float(parDim*perpDim), \ - float(nlc*nlp*state["NumThreads"]*state["GlobalReadVectorWidth%s"%tc]) * 100.0) \ - ) - - for p in range(0,nlp): - elementWidth = 4 - if p != nlp-1: - perp = state["LSP%s"%tc] - else: - perp = perpOverhang if perpOverhang else state["LSP%s"%tc] - - validElements = state["LSC%s"%tc] * perp - print(" buffer_load_element_x%u %ux%ux%u bytes, %u/%u valid GRO" %\ - (state["GlobalReadVectorWidth%s"%tc], \ - state["LSC%s"%tc], perp, \ - elementWidth, \ - validElements//state["GlobalReadVectorWidth%s"%tc], - state["NumThreads"])) - - return True - - - @staticmethod - def MatrixInstructionToMIParameters(state): - isa = tuple(state["ISA"]) - if len(state["MatrixInstruction"]) == 9: - mi = state["MatrixInstruction"] - state["MatrixInstruction"] = [state["MatrixInstruction"][0],state["MatrixInstruction"][1],state["MatrixInstruction"][2],state["MatrixInstruction"][3]] - - waves = mi[7]* mi[8] - miwg0 = mi[4] * mi[0] * mi[7] - state["WorkGroup"][0] = miwg0 - state["WorkGroup"][1] = waves*state["WavefrontSize"] // state["WorkGroup"][0] - state["ThreadTile"][0] = 1 # dummy - state["ThreadTile"][1] = 1 # dummy - - state["MFMA_BF16_1K"] = False - if not state["ProblemType"]["Sparse"]: - miDataType = state["ProblemType"]["DataType"] if (not state["EnableF32XdlMathOp"]) else state["ProblemType"]["F32XdlMathOp"] - if globalParameters["AsmCaps"][isa]["HasMFMA"]: - if not (miDataType.toChar() in validMFMA and \ - state["MatrixInstruction"] in validMFMA[miDataType.toChar()]): - if miDataType.isBFloat16() and \ - state["MatrixInstruction"] in validMFMA["B1k"]: - state["MFMA_BF16_1K"] = True - else: - reject(state, "MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], miDataType)) - elif globalParameters["AsmCaps"][isa]["HasWMMA"]: - if state["MatrixInstruction"] not in validWMMA: - reject(state, "MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], state["ProblemType"]["DataType"])) - else: - if not (state["ProblemType"]["DataType"].toChar() in validSMFMA and \ - state["MatrixInstruction"] in validSMFMA[state["ProblemType"]["DataType"].toChar()]): - reject(state, "Sparse MatrixInstruction %s not valid for DataType %s" % (state["MatrixInstruction"], state["ProblemType"]["DataType"])) - - # set EnableMatrixInstruction - state["EnableMatrixInstruction"] = True - - # set MIBlock - MIBlock_BM = miwg0 // mi[0] - MIBlock_BM = min(MIBlock_BM, mi[3]) - MIBlock_BN = mi[3] // MIBlock_BM - - state["MIBlock"] = [32, 32, 2, 1, 1, 1] - state["MIBlock"][0] = mi[0] - state["MIBlock"][1] = mi[1] - state["MIBlock"][2] = mi[2] - state["MIBlock"][3] = mi[3] - state["MIBlock"][4] = MIBlock_BM - state["MIBlock"][5] = MIBlock_BN - - # set MIWaveGroup - state['MIWaveGroup'] = [1, 1] - state['MIWaveGroup'][0] = min((miwg0 // mi[0]) // MIBlock_BM, waves) - state['MIWaveGroup'][1] = waves // state['MIWaveGroup'][0] - - # set MIWaveTile - state['MIWaveTile'] = [1, 1] - state['MIWaveTile'][0] = mi[5] - state['MIWaveTile'][1] = mi[6] - # set MIInputPerThread - isa = tuple(state["ISA"]) - state['MIInputPerThread'] = state["MatrixInstruction"][0] * state["MatrixInstruction"][2] * state["MatrixInstruction"][3] // state["WavefrontSize"] - if (not globalParameters["AsmCaps"][isa]['HasMFMA']) and globalParameters["AsmCaps"][isa]['HasWMMA']: - if state['ISA'][0] == 10 or state['ISA'][0] == 11: - state['MIInputPerThread'] = state["MatrixInstruction"][2] - sparseA = False if not state["ProblemType"]["Sparse"] else False if state["ProblemType"]["Sparse"] == 2 else True - sparseB = False if not state["ProblemType"]["Sparse"] else True if state["ProblemType"]["Sparse"] == 2 else False - state['MIInputPerThreadA'] = state['MIInputPerThread'] if not sparseA else state['MIInputPerThread']//2 - state['MIInputPerThreadB'] = state['MIInputPerThread'] if not sparseB else state['MIInputPerThread']//2 - state['MIInputPerThreadMetadata'] = state['MIInputPerThread'] if not state["ProblemType"]["Sparse"] else state['MIInputPerThread']//8 - elif state["MatrixInstruction"] != [] and len(state["MatrixInstruction"]) == 4: - state["EnableMatrixInstruction"] = True - else: - state["EnableMatrixInstruction"] = False ############################################## # check and calculate Wave Separate Global Read @staticmethod - def checkAndAssignWaveSeparateGlobalRead(state, tc): + def checkAndAssignWaveSeparateGlobalRead(state, tc, printRejectionReason: bool): # check can we use WaveSeparateGlobalRead numOfWaves = state["NumThreads"] // state["WavefrontSize"] if state["WaveSeparateGlobalRead%s"%tc]: if state["ProblemType"]["TLU%s"%tc] and (state["_DepthU%s"%tc] > 0) and (state["_DepthU%s"%tc] % numOfWaves != 0): - reject(state, "didn't support WaveSeparateGlobalRead when DepthU is not multiple of wave %u in TLU%s" % (state["_DepthU%s"%tc], tc)) + reject(state, printRejectionReason, "didn't support WaveSeparateGlobalRead when DepthU is not multiple of wave %u in TLU%s" % (state["_DepthU%s"%tc], tc)) if not state["ProblemType"]["TLU%s"%tc] and (state["MacroTile%s" % tc] % numOfWaves != 0): - reject(state, "didn't support WaveSeparateGlobalRead when MacroTile is not multiple of wave %u in TLU%s" % (state["MacroTile%s"%tc], tc)) + reject(state, printRejectionReason, "didn't support WaveSeparateGlobalRead when MacroTile is not multiple of wave %u in TLU%s" % (state["MacroTile%s"%tc], tc)) ######################################## # determine can we use VgprForLocalReadPacking @staticmethod - def isVgprForLocalReadPackingDoable(state): + def isVgprForLocalReadPackingDoable(state, isaInfoMap: Dict[str, IsaInfo]): isa = tuple(state["ISA"]) doable = True # MatrixInstruction only if not state["EnableMatrixInstruction"]: doable = False # only for HasEccHalf - if not globalParameters["ArchCaps"][isa]["HasEccHalf"]: + if not isaInfoMap[isa].archCaps["HasEccHalf"]: doable = False # only for PLR>=1 (except for DTVA+B) if state["PrefetchLocalRead"] < 1 and not (state["DirectToVgprA"] and state["DirectToVgprB"]): @@ -1837,19 +720,19 @@ def isDirectToVgprSupportDataType(state): ######################################## # determine can we use DirectToVgpr @staticmethod - def isDirectToVgprDoable(state, tc): + def isDirectToVgprDoable(state, tc, printRejectionReason: bool, isaInfoMap: Dict[str, IsaInfo]): MIindex = 0 if tc == 'A' else 1 numBytes = state["ProblemType"]["DataType"].numBytes() numBytesGR = state["ProblemType"]["DataType%s"%tc].numBytes() # With MatrixInstruction only if not state["EnableMatrixInstruction"] : - reject(state, "DirectToVgpr is for MatrixInstruction only") + reject(state, printRejectionReason, "DirectToVgpr is for MatrixInstruction only") return False # disable the following combinations for initial implementation # TODO: enable them if state["LocalSplitU"] != 1 and (not state["ProblemType"]["TLU%c"%tc]): - reject(state, "DirectToVgpr + LSU + TLU=False has not been enabled yet(tentative)") + reject(state, printRejectionReason, "DirectToVgpr + LSU + TLU=False has not been enabled yet(tentative)") return False if state["DirectToVgprA"] and state["DirectToVgprB"]: @@ -1860,42 +743,42 @@ def isDirectToVgprDoable(state, tc): state["PrefetchLocalRead"] = 0 # So far, DTVA + DTVB does not perform well (waitcnt is not ideal). # Disable it for now (TODO: improve waitcnt and re-enable) - reject(state, "DirectToVgprA + DirectToVgprB disabled") + reject(state, printRejectionReason, "DirectToVgprA + DirectToVgprB disabled") return False # DTV + input type conversion if state["ProblemType"]["DataType%s"%tc] != state["ProblemType"]["DataType"]: if not state["ConvertAfterDS"]: - reject(state, "DirectToVgpr%s + input conversion + ConvertAfterDS=False not supported"%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%s + input conversion + ConvertAfterDS=False not supported"%(tc)) return False # check if the DataType can support DirectToVgpr if not Solution.isDirectToVgprSupportDataType(state): - reject(state, "no DirectToVgpr support for this input data type") + reject(state, printRejectionReason, "no DirectToVgpr support for this input data type") return False # Does not work with TLU = False and PrefetchLocalRead = 0 if (not state["ProblemType"]["TLU%c"%tc]) and state["PrefetchLocalRead"] == 0: - reject(state, "DirectToVgpr%c does not supports TLU%c = False and PrefetchLocalRead = 0"%(tc, tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports TLU%c = False and PrefetchLocalRead = 0"%(tc, tc)) return False # Does not work with TLU = False and CGEMM/DGEMM/DGEMM (not supported) if (not state["ProblemType"]["TLU%c"%tc]) and (state["ProblemType"]["DataType"].isDouble() or \ state["ProblemType"]["DataType"].isComplex()): - reject(state, "DirectToVgpr%c does not supports TLU%c = False + S/C/D/ZGEMM"%(tc, tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports TLU%c = False + S/C/D/ZGEMM"%(tc, tc)) return False if numBytesGR * state["GlobalReadVectorWidth%c"%tc] < 4: # no support for DTV + numBytesGR * GlobalReadVectorWidth< 4 - reject(state, "DirectToVgpr%c does not support TLU%c + numByte * GlobalReadVectorWidth%c < 4"%(tc, tc, tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not support TLU%c + numByte * GlobalReadVectorWidth%c < 4"%(tc, tc, tc)) return False if numBytes < 4: # numBytes < 4 case if state["ProblemType"]["TLU%c"%tc]: # use pack logic (with v_perm) same as local read (only if VgprForLocalReadPacking is doable) - if not Solution.isVgprForLocalReadPackingDoable(state): - reject(state, "Does not meet the requirement for DirectToVgpr%c + TLU%c + numByte < 4"%(tc, tc)) + if not Solution.isVgprForLocalReadPackingDoable(state, isaInfoMap): + reject(state, printRejectionReason, "Does not meet the requirement for DirectToVgpr%c + TLU%c + numByte < 4"%(tc, tc)) return False # force ClusterLocalRead=1 for DTV + pack state["ClusterLocalRead"] = 1 @@ -1903,7 +786,7 @@ def isDirectToVgprDoable(state, tc): # numBytes >= 4 case if state["ProblemType"]["TLU%c"%tc] and state["MIInputPerThread"] > 1: # no support for numBytes >= 4 + MIInputPerThread > 1 - reject(state, "DirectToVgpr%c does not support TLU%c+ numByte >= 4 + MIInputPerThread > 1"%(tc, tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not support TLU%c+ numByte >= 4 + MIInputPerThread > 1"%(tc, tc)) return False # MatrixInstBM,BN check @@ -1911,52 +794,52 @@ def isDirectToVgprDoable(state, tc): # for B, MatrixInstBM should be 1 # This is to limit the number of Vgpr if tc == 'A' and not (state['MatrixInstBN'] == 1): - reject(state, "MatrixInstBN should be 1 for DirectToVgprA. Current value is %d"%(state['MatrixInstBN'])) + reject(state, printRejectionReason, "MatrixInstBN should be 1 for DirectToVgprA. Current value is %d"%(state['MatrixInstBN'])) return False if tc == 'B' and not (state['MatrixInstBM'] == 1): - reject(state, "MatrixInstBM should be 1 for DirectToVgprB. Current value is %d"%(state['MatrixInstBM'])) + reject(state, printRejectionReason, "MatrixInstBM should be 1 for DirectToVgprB. Current value is %d"%(state['MatrixInstBM'])) return False # Does not work with WaveSeparateGlobalRead if state["WaveSeparateGlobalRead%c"%tc]: - reject(state, "DirectToVgpr%c does not supports WaveSeparateGlobalRead%c"%(tc, tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports WaveSeparateGlobalRead%c"%(tc, tc)) return False # Does not work with TLU + VectorWidth != GlobalReadVectorWidth (VW = 2 + GRVW = 1 or VW = 1 + GRVW = 2 does not work) if state["ProblemType"]["TLU%c"%tc] and state["VectorWidth%s"%tc] != state["GlobalReadVectorWidth%c"%tc]: - reject(state, "DirectToVgpr%c does not supports TLU + VectorWidth%s(=%u) != GlobalReadVectorWidth%c(%u)"%(tc, tc, state["VectorWidth%s"%tc], tc, state["GlobalReadVectorWidth%c"%tc])) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports TLU + VectorWidth%s(=%u) != GlobalReadVectorWidth%c(%u)"%(tc, tc, state["VectorWidth%s"%tc], tc, state["GlobalReadVectorWidth%c"%tc])) return False # Does not work with TLU=False and NumLoadsCoalesced != DepthU//(MatrixInstK*GRVW*LSU//MIInputPerThread) if (not state["ProblemType"]["TLU%c"%tc]) and \ state["NumLoadsCoalesced%c"%tc] != state["DepthU"] // (state["MatrixInstK"] * state["GlobalReadVectorWidth%c"%tc] * state["LocalSplitU"] // state["MIInputPerThread"]): - reject(state, "DirectToVgpr%c does not supports TLU=False and NumLoadsCoalesced%c != DepthU//(MatrixInstK*GlobalReadVectorWidth*LocalSplitU//MIInputPerThread(=%u))"%(tc, tc, state["MIInputPerThread"])) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports TLU=False and NumLoadsCoalesced%c != DepthU//(MatrixInstK*GlobalReadVectorWidth*LocalSplitU//MIInputPerThread(=%u))"%(tc, tc, state["MIInputPerThread"])) return False # TLU=False case, need GlobalReadVectorWidth == LocalReadVectorWidth if (not state["ProblemType"]["TLU%c"%tc]) and \ state["GlobalReadVectorWidth%c"%tc] != state["LocalReadVectorWidth"]: - reject(state, "DirectToVgpr%c does not supports TLU=False GlobalReadVectorWidth%c(%u) != LocalReadVectorWidth(%u)"%(tc, tc, state["GlobalReadVectorWidth%c"%tc], state["LocalReadVectorWidth"])) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports TLU=False GlobalReadVectorWidth%c(%u) != LocalReadVectorWidth(%u)"%(tc, tc, state["GlobalReadVectorWidth%c"%tc], state["LocalReadVectorWidth"])) return False # Does not work with SIA<3 if state["ScheduleIterAlg"] < 3: - reject(state, "DirectToVgpr%c does not supports ScheduleIterAlg < 3"%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports ScheduleIterAlg < 3"%(tc)) return False # Does not work with InnerUnroll>1 if state["InnerUnroll"]>1: - reject(state, "DirectToVgpr%c does not supports InnerUnroll>1"%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports InnerUnroll>1"%(tc)) return False # Reject TLU = UnrollMajorLDS if state["ProblemType"]["TLU%c"%tc] == state["UnrollMajorLDS%c"%tc]: - reject(state, "DirectToVgpr%c does not supports TLU%c = UnrollMajorLDS%c"%(tc, tc, tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports TLU%c = UnrollMajorLDS%c"%(tc, tc, tc)) return False # does not work with UnrollLoopSwapGlobalReadOrder if state["UnrollLoopSwapGlobalReadOrder"]: - reject(state, "DirectToVgpr%c does not supports UnrollLoopSwapGlobalReadOrder"%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports UnrollLoopSwapGlobalReadOrder"%(tc)) return False # does not work with PGR2 + EPS @@ -1966,17 +849,17 @@ def isDirectToVgprDoable(state, tc): # does not work with Sparse if state["ProblemType"]["Sparse"]: - reject(state, "DirectToVgpr%c does not supports Sparse"%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports Sparse"%(tc)) return False # for DTVA/DTVB, does not work with PGR0 if state["PrefetchGlobalRead"] == 0: - reject(state, "DirectToVgpr%c does not supports PrefetchGlobalRead == 0."%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports PrefetchGlobalRead == 0."%(tc)) return False # for DTVA, does not work with NN and TLDS0 if tc == 'A' and state["TransposeLDS"] == 0 and (not state["ProblemType"]["TransposeA"] and not state["ProblemType"]["TransposeB"]): - reject(state, "DirectToVgpr%c does not supports NN case with TransposeLDS == 0."%(tc)) + reject(state, printRejectionReason, "DirectToVgpr%c does not supports NN case with TransposeLDS == 0."%(tc)) return False # for DTVA, does not work with TT and Tail-loop @@ -1997,7 +880,7 @@ def isDirectToVgprDoable(state, tc): ######################################## # determine can we use DirectToLds @staticmethod - def isDirectToLdsDoable(state, tc): + def isDirectToLdsDoable(state, tc, printRejectionReason: bool): # x2/x4 support for directToLds (no longer supported) # numelements_perlane = 4/numBytes @@ -2018,85 +901,85 @@ def isDirectToLdsDoable(state, tc): # numBytesPerLoad == 4 only if numBytesPerLoad != 4: - reject(state, "DirectToLds can only be used with buffer loads requiring 1 register") + reject(state, printRejectionReason, "DirectToLds can only be used with buffer loads requiring 1 register") return False # so far MFMA only (TODO: enable non MFMA case) if not state["EnableMatrixInstruction"]: - reject(state, "DirectToLds is for MatrixInstruction only for now (tentative)") + reject(state, printRejectionReason, "DirectToLds is for MatrixInstruction only for now (tentative)") return False # so far, DirectToLds does not work with StreamK (TODO: enable StreamK case) if state["StreamK"]: - reject(state, "DirectToLds does not support StreamK (tentative)") + reject(state, printRejectionReason, "DirectToLds does not support StreamK (tentative)") return False # DTL + LocalReadVectorWidth > MIInputPerThread does not work # Need support for TailLoop if state["LocalReadVectorWidth"] > state["MIInputPerThread"]: - reject(state, "DirectToLds does not work with LocalReadVectorWidth > MIInputPerThread") + reject(state, printRejectionReason, "DirectToLds does not work with LocalReadVectorWidth > MIInputPerThread") return False if state["AssertSummationElementMultiple"] % state["GlobalReadVectorWidth%c"%tc] != 0: - reject(state, "can't use DirectToLds with AssertSummationElementMultiple(%u) %% GlobalReadVectorWidth%c(%u)" % \ + reject(state, printRejectionReason, "can't use DirectToLds with AssertSummationElementMultiple(%u) %% GlobalReadVectorWidth%c(%u)" % \ (state["AssertSummationElementMultiple"], tc, state["GlobalReadVectorWidth%c"%tc])) return False if state["NumThreads"] % state["WavefrontSize"] != 0: - reject(state, "can't use DirectToLds for NumThreads % WavefrontSize != 0") + reject(state, printRejectionReason, "can't use DirectToLds for NumThreads % WavefrontSize != 0") return False if state["ProblemType"]["TLU%c"%tc] == state["UnrollMajorLDS%c" % tc]: - reject(state, "can't use DirectToLds for TLU%c == UnrollMajorLDS%c"%(tc, tc)) + reject(state, printRejectionReason, "can't use DirectToLds for TLU%c == UnrollMajorLDS%c"%(tc, tc)) return False # avoid picking x2&x4 for precisions < f32/f64 in [ProblemType][TLU] == TRUE if not state["EnableMatrixInstruction"]: if state["GlobalReadVectorWidth%c"%tc] * numBytesAB * state["WavefrontSize"] > 256: - reject(state, "can't use DirectToLds for not EnableMatrixInstruction and GlobalReadVectorWidth%c * bpe%c * WavefrontSize > 256"%(tc,tc)) + reject(state, printRejectionReason, "can't use DirectToLds for not EnableMatrixInstruction and GlobalReadVectorWidth%c * bpe%c * WavefrontSize > 256"%(tc,tc)) return False if state["WaveSeparateGlobalRead%c" % tc]: if state["LSC%c"%tc] * state["LSP%c"%tc] * numBytesAB != state["WavefrontSize"] * state["GlobalReadVectorWidth%c"%tc] * numBytesAB: - reject(state, "can't use DirectToLds for LSC%c and LSP%c * bpe!= WavefrontSize * GlobalReadVectorWidth%c * bpe%c > 4"%(tc, tc, tc, tc)) + reject(state, printRejectionReason, "can't use DirectToLds for LSC%c and LSP%c * bpe!= WavefrontSize * GlobalReadVectorWidth%c * bpe%c > 4"%(tc, tc, tc, tc)) return False else: if state["LSC%c"%tc] * state["LSP%c"%tc] * numBytesAB != state["NumThreads"] * state["GlobalReadVectorWidth%c"%tc] * numBytesAB: - reject(state, "can't use DirectToLds for LSC%c and LSP%c * bpe != NumThreads * GlobalReadVectorWidth%c * bpe%c > 4"%(tc, tc, tc, tc)) + reject(state, printRejectionReason, "can't use DirectToLds for LSC%c and LSP%c * bpe != NumThreads * GlobalReadVectorWidth%c * bpe%c > 4"%(tc, tc, tc, tc)) return False # so far, DirectToLds does not work well with PGR=2 # performance is not good and a lot of ds_read for DTL can cause scheduling issue(need fix) if state["PrefetchGlobalRead"] == 2: - reject(state, "can't use DirectToLds for PrefetchGlobalRead == 2") + reject(state, printRejectionReason, "can't use DirectToLds for PrefetchGlobalRead == 2") return False # so far, DirectToLds does not work with LRVW=2 if state["LocalReadVectorWidth"] == 2: - reject(state, "can't use DirectToLds for LocalReadVectorWidth == 2") + reject(state, printRejectionReason, "can't use DirectToLds for LocalReadVectorWidth == 2") return False # Does not work with (NumLoadsCoalesced>1 and UseInstOffsetForGRO) + DGEMM if state["ProblemType"]["DataType"].isDouble() and \ (state["NumLoadsCoalesced%c"%tc] > 1 and state["UseInstOffsetForGRO"]): - reject(state, "DirectToLds%c does not supports NumLoadsCoalesced%c > 1 and UseInstOffsetForGRO for dgemm"%(tc, tc)) + reject(state, printRejectionReason, "DirectToLds%c does not supports NumLoadsCoalesced%c > 1 and UseInstOffsetForGRO for dgemm"%(tc, tc)) return False # Does not work with NumLoadsCoalesced>1 + ZGEMM if state["ProblemType"]["DataType"].isDoubleComplex() and state["NumLoadsCoalesced%c"%tc] > 1: - reject(state, "DirectToLds%c does not supports NumLoadsCoalesced%c > 1 for zgemm"%(tc, tc)) + reject(state, printRejectionReason, "DirectToLds%c does not supports NumLoadsCoalesced%c > 1 for zgemm"%(tc, tc)) return False # Does not work with PrefetchGlobalRead=2 and PrefetchLocalRead=1 (cannot schedule DTL global read after local read) if state["PrefetchGlobalRead"] == 2 and state["PrefetchLocalRead"] == 1: - reject(state, "DirectToLds%c does not work with PrefetchGlobalRead=2 and PrefetchLocalRead=1"%(tc)) + reject(state, printRejectionReason, "DirectToLds%c does not work with PrefetchGlobalRead=2 and PrefetchLocalRead=1"%(tc)) return False # DirectToLds does not work if MacroTile is not power of 2 # LDS offset swap/rotate logic works only when MacroTile is power of 2 mt = state["MacroTile%c"%tc] if mt & (mt - 1) != 0: - reject(state, "can't use DirectToLds if MacroTile%s is not power of 2"%tc) + reject(state, printRejectionReason, "can't use DirectToLds if MacroTile%s is not power of 2"%tc) return False # DirectToLds does not work with TLU=False and bpe > bpr and DepthU//NumLoadsCoalesced < 8 @@ -2105,7 +988,7 @@ def isDirectToLdsDoable(state, tc): # current offset swap logic does not work if (not state["ProblemType"]["TLU%c"%tc]) and state["ProblemType"]["DataType"].numRegisters() > 1 and \ state["_DepthU%s"%tc] // state["NumLoadsCoalesced%c"%tc] < 8: - reject(state, "DirectToLds%c does not work with TLU=False and bpe > bpr and DepthU//NumLoadsCoalesced%c < 8"%(tc, tc)) + reject(state, printRejectionReason, "DirectToLds%c does not work with TLU=False and bpe > bpr and DepthU//NumLoadsCoalesced%c < 8"%(tc, tc)) return False return True @@ -2128,15 +1011,23 @@ def getDivisorName(state, tC): ######################################## # assign all derived parameters @staticmethod - def assignDerivedParameters(state): - state["EnableF32XdlMathOp"] = False #ignore the F32 xDL MathOp by default. - #enable F32 xDL MathOp only when the input type is f32. - if "F32XdlMathOp" in state["ProblemType"] \ - and (not state["ProblemType"]["F32XdlMathOp"].isSingle()) \ - and (state["ProblemType"]["DataType"].isSingle()): - state["EnableF32XdlMathOp"] = True - - Solution.assignProblemIndependentDerivedParameters(state) + def assignDerivedParameters( + state, + splitGSU: bool, + printRejectionReason: bool, + printIndexAssignmentInfo: bool, + isaInfoMap, + rocmVersion: SemanticVersion, + depthUConfig: DepthUConfig + ): + # state["EnableF32XdlMathOp"] = False #ignore the F32 xDL MathOp by default. + # #enable F32 xDL MathOp only when the input type is f32. + # if "F32XdlMathOp" in state["ProblemType"] \ + # and (not state["ProblemType"]["F32XdlMathOp"].isSingle()) \ + # and (state["ProblemType"]["DataType"].isSingle()): + # state["EnableF32XdlMathOp"] = True + + Solution.assignProblemIndependentDerivedParameters(state, printRejectionReason, isaInfoMap) if "AssignedDerivedParameters" in state: if state["AssignedDerivedParameters"]: @@ -2160,7 +1051,7 @@ def assignDerivedParameters(state): elif state["GlobalSplitUAlgorithm"] == 'MultipleBuffer': state["_GlobalAccumulation"] = 'MultipleBuffer' elif state["GlobalSplitUAlgorithm"] == 'MultipleBufferSingleKernel': - if (not globalParameters["SplitGSU"]): + if (not splitGSU): state["_GlobalAccumulation"] = 'MultipleBufferSingleKernel' else: if state["GlobalSplitU"] > 1: @@ -2176,32 +1067,32 @@ def assignDerivedParameters(state): state["GlobalSplitU"] = 0 # Cannot enable both Stream-K and GSU state["GlobalSplitUAlgorithm"] = "MultipleBuffer" # Set default Algorithm if state["ProblemType"]["DataType"].isDouble(): - reject(state, "Type {} for DataType not yet supported with StreamK".format(state["ProblemType"]["DataType"].toChar())) + reject(state, printRejectionReason, "Type {} for DataType not yet supported with StreamK".format(state["ProblemType"]["DataType"].toChar())) if state["MIWaveGroup"][0] * state["MIWaveGroup"][1] != 4: - reject(state, "Stream-K requries MIWaveGroup0*MIWaveGroup1=4") + reject(state, printRejectionReason, "Stream-K requries MIWaveGroup0*MIWaveGroup1=4") if not state["EnableMatrixInstruction"]: - reject(state, "Stream-K requires MatrixInstruction") - if globalParameters["AsmCaps"][isa]["HasWMMA"]: - reject(state, "Stream-K untested with WMMA") + reject(state, printRejectionReason, "Stream-K requires MatrixInstruction") + if isaInfoMap[isa].asmCaps["HasWMMA"]: + reject(state, printRejectionReason, "Stream-K untested with WMMA") # if state["PersistentKernel"]: - # reject(state, "Cannot enable both Stream-K and PersistentKernel") + # reject(state, printRejectionReason, "Cannot enable both Stream-K and PersistentKernel") if not state["ProblemType"]["StridedBatched"]: - reject(state, "General batch not supported with Stream-K") + reject(state, printRejectionReason, "General batch not supported with Stream-K") if state["ProblemType"]["GroupedGemm"]: - reject(state, "Grouped gemm not yet supported with Stream-K") + reject(state, printRejectionReason, "Grouped gemm not yet supported with Stream-K") if state["ScheduleGlobalRead"] != 1: - reject(state, "ScheduleGlobalRead not supported with Stream-K") + reject(state, printRejectionReason, "ScheduleGlobalRead not supported with Stream-K") if state["ScheduleLocalWrite"] != 1: - reject(state, "ScheduleLocalWrite not supported with Stream-K") + reject(state, printRejectionReason, "ScheduleLocalWrite not supported with Stream-K") if state["ScheduleIterAlg"] != 2 and state["ScheduleIterAlg"] != 3: - reject(state, "ScheduleIterAlg not supported with Stream-K") + reject(state, printRejectionReason, "ScheduleIterAlg not supported with Stream-K") if state["StreamKAtomic"] == 1: if not state["ProblemType"]["DataType"].isSingle(): - reject(state, "Atomic Stream-K currently only tested for SGEMM") + reject(state, printRejectionReason, "Atomic Stream-K currently only tested for SGEMM") if not state["BufferStore"]: - reject(state, "Atomic Stream-K requires BufferStore") + reject(state, printRejectionReason, "Atomic Stream-K requires BufferStore") if state["LocalSplitU"] > 1: - reject(state, "Atomic Stream-K not working with LocalSplitU") + reject(state, printRejectionReason, "Atomic Stream-K not working with LocalSplitU") if not state["Valid"]: return else: @@ -2221,28 +1112,28 @@ def assignDerivedParameters(state): if state["VectorStore"] == -1: state["_VectorStore"] = 1 # default, may be changed if needed to generate a valid kernel - ProblemType.assignDerivedParameters(state["ProblemType"]) + ProblemType.assignDerivedParameters(state["ProblemType"], printIndexAssignmentInfo) if not state["Valid"]: print2("in assignDerivedParameters, state['Valid'] = False") return - if not globalParameters["AsmCaps"][isa]["HasNTModifier"]: + if not isaInfoMap[isa].asmCaps["HasNTModifier"]: # force to disable nt flag if it is not supported by arch for ch in ["", "A", "B", "C", "D", "E", "WS", "Metadata"]: if state["NonTemporal%s"%ch] >= 4: state["NonTemporal%s"%ch] -= 4 - if state["WavefrontSize"] == 32 and not globalParameters["ArchCaps"][isa]["HasWave32"]: - reject(state, "WavefrontSize=32 not supported for ISA {}".format(isa)) + if state["WavefrontSize"] == 32 and not isaInfoMap[isa].archCaps["HasWave32"]: + reject(state, printRejectionReason, "WavefrontSize=32 not supported for ISA {}".format(isa)) return if state["WavefrontSize"] == 32 and state["KernelLanguage"] == "Source": - reject(state, "WavefrontSize=32 not yet supported for source kernels.") + reject(state, printRejectionReason, "WavefrontSize=32 not yet supported for source kernels.") return if state["EnableMatrixInstruction"]: - if not (globalParameters["AsmCaps"][isa]["HasMFMA"] or globalParameters["AsmCaps"][isa]["HasWMMA"]): - reject(state, f"isa {isa} doesn't support matrix instruction") + if not (isaInfoMap[isa].asmCaps["HasMFMA"] or isaInfoMap[isa].asmCaps["HasWMMA"]): + reject(state, printRejectionReason, f"isa {isa} doesn't support matrix instruction") return if not (state["ProblemType"]["DataType"].isSingle() \ or state["ProblemType"]["DataType"].isDouble() \ @@ -2251,62 +1142,62 @@ def assignDerivedParameters(state): or state["ProblemType"]["DataType"].isComplex() \ or state["ProblemType"]["DataType"].is8bitFloat() \ or state["ProblemType"]["DataType"].isInt8()): - reject(state, "didn't support Matrix Instruction with type %s" % str(state["ProblemType"]["DataType"])) + reject(state, printRejectionReason, "didn't support Matrix Instruction with type %s" % str(state["ProblemType"]["DataType"])) return - if (not globalParameters["AsmCaps"][isa]["HasMFMA"] and globalParameters["AsmCaps"][isa]["HasWMMA"] and (state["WavefrontSize"] == 64)): + if (not isaInfoMap[isa].asmCaps["HasMFMA"] and isaInfoMap[isa].asmCaps["HasWMMA"] and (state["WavefrontSize"] == 64)): print2("!! Warning: WMMA only well tune on WGP mode, wave size = 32") - # reject(state, "WMMA only suppport on WGP mode, wave size = 32") + # reject(state, printRejectionReason, "WMMA only suppport on WGP mode, wave size = 32") # return if not state["MIBlock"] or len(state["MIBlock"]) != 6: - reject(state, "invalid MIBlock") + reject(state, printRejectionReason, "invalid MIBlock") return if not state["MIWaveGroup"] or len(state["MIWaveGroup"]) != 2: - reject(state, "invalid MIWaveGroup") + reject(state, printRejectionReason, "invalid MIWaveGroup") return if not state["MIWaveTile"] or len(state["MIWaveTile"]) != 2: - reject(state, "invalid MIWaveTile") + reject(state, printRejectionReason, "invalid MIWaveTile") return - if globalParameters["AsmCaps"][isa]["HasMFMA"]: + if isaInfoMap[isa].asmCaps["HasMFMA"]: if not state["ProblemType"]["HighPrecisionAccumulate"] \ and state["ProblemType"]["DataType"].numRegisters() < 1 : - reject(state, "Matrix instructions for half, bf16 (or i8) types are natively accumulated" + \ + reject(state, printRejectionReason, "Matrix instructions for half, bf16 (or i8) types are natively accumulated" + \ " in fp32 (or i32) precision. Please add the following config:" + \ "\n - HighPrecisionAccumulate: True") return - if globalParameters["AsmCaps"][isa]["HasWMMA"]: + if isaInfoMap[isa].asmCaps["HasWMMA"]: if state["ProblemType"]["DataType"].numRegisters() >=1: - reject(state, "WMMA only support half, bf16 and i8 type") + reject(state, printRejectionReason, "WMMA only support half, bf16 and i8 type") return if state["InterleaveAlpha"]: - reject(state, "Matrix instruction doesn't support InterleaveAlpha") + reject(state, printRejectionReason, "Matrix instruction doesn't support InterleaveAlpha") return if state["ProblemType"]["DataType"].isInt8(): if isa[:2] == (9, 4): if tuple(state["MatrixInstruction"])[:3] in ((32, 32, 8), (16, 16, 16)): - reject(state, "v_mfma_i32_32x32x8 and v_mfma_i32_16x16x16 have been deprecated in gfx94x") + reject(state, printRejectionReason, "v_mfma_i32_32x32x8 and v_mfma_i32_16x16x16 have been deprecated in gfx94x") return if state["ProblemType"]["ComputeDataType"].isDouble(): # See [4,4,4,4] snop for more info - if state["MatrixInstruction"] == [4,4,4,4] and (not state['ISA'] == [9,0,10]) and state["ScheduleIterAlg"] == 3: - reject(state, "Currently Matrix instructions [4,4,4,4] is disabled.") + if state["MatrixInstruction"] == [4,4,4,4] and (not state['ISA'] == IsaVersion(9,0,10)) and state["ScheduleIterAlg"] == 3: + reject(state, printRejectionReason, "Currently Matrix instructions [4,4,4,4] is disabled.") return else: if not state["ProblemType"]["HighPrecisionAccumulate"] \ and state["ProblemType"]["ComputeDataType"].numRegisters() > state["ProblemType"]["DataType"].numRegisters() : - reject(state, "For non-MI Kernel, if sizeof(ComputeDataType) > sizeof(DataType), " + \ + reject(state, printRejectionReason, "For non-MI Kernel, if sizeof(ComputeDataType) > sizeof(DataType), " + \ "Please add the following config:" + \ "\n - HighPrecisionAccumulate: True") return if state["ProblemType"]["Sparse"]: - reject(state, "Sparse A problem is only supported by SMFMA MI kernel.") + reject(state, printRejectionReason, "Sparse A problem is only supported by SMFMA MI kernel.") return if state["ThreadTile0"] > 16 or state["ThreadTile1"] > 16: - reject(state, "Invalid value for ThreadTile") + reject(state, printRejectionReason, "Invalid value for ThreadTile") return if state["ScheduleIterAlg"] == 2 or state["ScheduleIterAlg"] == 3: - reject(state, "SIA2 and SIA3 only support MatrixInstruction") + reject(state, printRejectionReason, "SIA2 and SIA3 only support MatrixInstruction") return if state["ProblemType"]["Tensor0"]==0: @@ -2331,7 +1222,7 @@ def assignDerivedParameters(state): state["MIWaveTileB"] = state["MIWaveTile"][0] if state["ProblemType"]["Sparse"] == 2 and state["DirectToVgprSparseMetadata"]: - reject(state, "Sparse B does not supprot DirectToVgprSparseMetadata") + reject(state, printRejectionReason, "Sparse B does not supprot DirectToVgprSparseMetadata") return @@ -2384,7 +1275,7 @@ def assignDerivedParameters(state): for (tc,batchMask) in (('A', 0x1), ('B', 0x2)): freeDims = [i for i in problemType["IndexAssignments%s"%tc] if i in problemType["IndicesFree"]] if not freeDims: - reject(state, "tensor%s contains no free indices.") + reject(state, printRejectionReason, "tensor%s contains no free indices.") return False # Determine which indices will be packed together as this impacts several different parms (sizes, magic numbers, etc) @@ -2435,11 +1326,11 @@ def assignDerivedParameters(state): state["DirectToLds"] = False state["_UseSgprForGRO"] = False if state["PrefetchGlobalRead"] == 2: - reject(state, "BufferLoad=0 does not support PrefetchGlobalRead=2") + reject(state, printRejectionReason, "BufferLoad=0 does not support PrefetchGlobalRead=2") return if problemType["UseBias"]: - reject(state, "BufferLoad=0 does not support UseBias due to no suppress no load.") + reject(state, printRejectionReason, "BufferLoad=0 does not support UseBias due to no suppress no load.") return #These modes only work under certain conditions, apply them here: @@ -2506,15 +1397,15 @@ def assignDerivedParameters(state): state["VectorWidthMetadata"] = state["VectorWidthA"] if state["ProblemType"]["Sparse"] == 1 else state["VectorWidthB"] # if state["EnableMatrixInstruction"] and not state["SourceSwap"] and (state["VectorWidthA"] > 1 or state["VectorWidthB"] > 1): - # reject(state, "not implement VectorWidth without SourceSwap") + # reject(state, printRejectionReason, "not implement VectorWidth without SourceSwap") # TT0,1 both must be multiples of VW, b/c of rC, rA, rB if state["EnableMatrixInstruction"]: if (state["MIWaveTile"][0] % state["VectorWidthA"]) != 0: - reject(state, "MIWaveTile0(%u) should be multiple of VectorWidthA(%u)" % (state["MIWaveTile"][0], state["VectorWidthA"])) + reject(state, printRejectionReason, "MIWaveTile0(%u) should be multiple of VectorWidthA(%u)" % (state["MIWaveTile"][0], state["VectorWidthA"])) return if (state["MIWaveTile"][1] % state["VectorWidthB"]) != 0: - reject(state, "MIWaveTile0(%u) should be multiple of VectorWidthB(%u)" % (state["MIWaveTile"][1], state["VectorWidthB"])) + reject(state, printRejectionReason, "MIWaveTile0(%u) should be multiple of VectorWidthB(%u)" % (state["MIWaveTile"][1], state["VectorWidthB"])) return if len(problemType["IndicesSummation"]) > 1: @@ -2526,18 +1417,18 @@ def assignDerivedParameters(state): if state["KernelLanguage"] == "Assembly" \ and state["ProblemType"]["DataType"].isHalf(): - if globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"]: + if isaInfoMap[state["ISA"]].archCaps["HasEccHalf"]: if not state["ProblemType"]["HighPrecisionAccumulate"] and state["AssertFree0ElementMultiple"] % 2 != 0: # beta-on-edge has AF0EM requirement except for HPA kernels - reject(state, "Archs with HasEccHalf require AF0EM%2==0 except for HPA kernels") + reject(state, printRejectionReason, "Archs with HasEccHalf require AF0EM%2==0 except for HPA kernels") return if state["ConvertAfterDS"]: if (state["ProblemType"]["DataType"].isHalf() == False): - reject(state, "ConvertAfterDS only support DataType half") + reject(state, printRejectionReason, "ConvertAfterDS only support DataType half") return if (state["ProblemType"]["DataTypeA"].isAnyFloat8() == False) and (state["ProblemType"]["DataTypeB"].isAnyFloat8() == False): - reject(state, "one of DataTypeA or DataTypeB need to be float8") + reject(state, printRejectionReason, "one of DataTypeA or DataTypeB need to be float8") return # DepthU == -1? @@ -2558,7 +1449,20 @@ def assignDerivedParameters(state): state[backup[0]] = backup[1] state["ValidDepthU"] = True state["DepthU"] = depthuList[index[0]] - Solution.depthUIteration(state, index, depthuList, problemType, isa, bufferLoad, packedC0, packedC1) + Solution.depthUIteration( + state, + index, + depthuList, + problemType, + isa, + bufferLoad, + packedC0, + packedC1, + printRejectionReason, + isaInfoMap, + rocmVersion, + depthUConfig, + ) if state["Valid"] or (state["ValidDepthU"] and (not state["Valid"])): break index[0] += 1 @@ -2566,8 +1470,21 @@ def assignDerivedParameters(state): break if "ValidDepthU" in state: del state["ValidDepthU"] - - def depthUIteration(state, index, depthuList, problemType, isa, bufferLoad, packedC0, packedC1): + + def depthUIteration( + state, + index, + depthuList, + problemType, + isa, + bufferLoad, + packedC0, + packedC1, + printRejectionReason: bool, + isaInfoMap: Dict[IsaVersion, IsaInfo], + rocmVersion: SemanticVersion, + depthUConfig: DepthUConfig + ): ######################################## # Auto search for DepthU starts here # Activates when DepthU == -1 @@ -2601,15 +1518,15 @@ def depthUIteration(state, index, depthuList, problemType, isa, bufferLoad, pack state["_DepthUB"] = depthUB# internal state["_DepthUMetadata"] = depthUM# internal - Solution.checkAndAssignWaveSeparateGlobalRead(state, 'A') - Solution.checkAndAssignWaveSeparateGlobalRead(state, 'B') + Solution.checkAndAssignWaveSeparateGlobalRead(state, 'A', printRejectionReason) + Solution.checkAndAssignWaveSeparateGlobalRead(state, 'B', printRejectionReason) if state["ProblemType"]["Sparse"]: if state["ProblemType"]["Sparse"] == 2: if not state["DirectToVgprSparseMetadata"]: - Solution.checkAndAssignWaveSeparateGlobalRead(state, 'Metadata') + Solution.checkAndAssignWaveSeparateGlobalRead(state, 'Metadata', printRejectionReason) else: if not state["DirectToVgprSparseMetadata"]: - Solution.checkAndAssignWaveSeparateGlobalRead(state, 'Metadata') + Solution.checkAndAssignWaveSeparateGlobalRead(state, 'Metadata', printRejectionReason) # Set up stagger shift: bpeAB = int(4*state["ProblemType"]["DataType"].numRegisters()) @@ -2627,12 +1544,12 @@ def depthUIteration(state, index, depthuList, problemType, isa, bufferLoad, pack except ValueError: # i.e., StaggerUStride == 0 staggerStrideShift = 0 if staggerStrideShift < 0: - reject(state, "StaggerUStride=%u is less than size of DepthU=%u * BytesPerElement=%u" \ + reject(state, printRejectionReason, "StaggerUStride=%u is less than size of DepthU=%u * BytesPerElement=%u" \ % (state["StaggerUStride"], state["DepthU"], bpeAB)) #print "staggerStrideShift=", staggerStrideShift, "depthu=", state["DepthU"] state["_staggerStrideShift"] = staggerStrideShift - def calcLdsPad(lrvw: int) -> int: + def calcLdsPad(lrvw: int, isaInfoMap: Dict[str, IsaInfo]) -> int: ldsPadA = state["LdsPadA"] ldsPadB = state["LdsPadB"] optPadA = optPadB = lrvw @@ -2644,7 +1561,7 @@ def calcLdsPad(lrvw: int) -> int: else: optPadA //= 2 readRegsA //= 2 - if (not globalParameters["AsmCaps"][isa]['HasWMMA']) and (readRegsA > 4 or readRegsB > 4): + if (not isaInfoMap[isa].asmCaps['HasWMMA']) and (readRegsA > 4 or readRegsB > 4): reject(state, "LocalReadVectorWidth results in attemping to read LDS larger than b128, reject") return if state["EnableMatrixInstruction"]: @@ -2822,9 +1739,9 @@ def calcLdsNumBytes(ldsPadA: int, LdsBlockSizePerPadA: int, ldsPadB: int, LdsBlo state["LocalReadVectorWidth"] = state["MIInputPerThread"] else: if state["LocalReadVectorWidth"] < state["MIInputPerThread"]: - reject(state, "LocalReadVectorWidth < %u" %(state["MIInputPerThread"])) + reject(state, printRejectionReason, "LocalReadVectorWidth < %u" %(state["MIInputPerThread"])) if state["LocalReadVectorWidth"] > state["MIInputPerThread"] and not state["TransposeLDS"]: - reject(state, "LocalReadVectorWidth require Transpose LDS") + reject(state, printRejectionReason, "LocalReadVectorWidth require Transpose LDS") if autoLRVW: if state["LocalReadVectorWidth"] // state["MIInputPerThread"] > 1: @@ -2832,10 +1749,10 @@ def calcLdsNumBytes(ldsPadA: int, LdsBlockSizePerPadA: int, ldsPadB: int, LdsBlo # if only have 1 iteration with wider local read, reduce LRVW to have better scheduling (at least 2 iterations) state["LocalReadVectorWidth"] //= 2 if state["LocalReadVectorWidth"] // state["MIInputPerThread"] > 1: - padA, padB, padM = calcLdsPad(state["LocalReadVectorWidth"]) + padA, padB, padM = calcLdsPad(state["LocalReadVectorWidth"], isaInfoMap) ldsBlockSizePerPadA, ldsBlockSizePerPadB = calcLdsBlockSizePerPad(state["LocalReadVectorWidth"]) ldsNumBytesA, ldsNumBytesAlignedA, ldsNumBytesB, ldsNumBytesAlignedB, ldsNumBytesMetadata, ldsNumBytesAlignedMetadata = calcLdsNumBytes(padA, ldsBlockSizePerPadA, padB, ldsBlockSizePerPadB) - if (ldsNumBytesAlignedA + ldsNumBytesAlignedB) > globalParameters["MaxLDS"]: + if (ldsNumBytesAlignedA + ldsNumBytesAlignedB) > depthUConfig.maxLDS: state["LocalReadVectorWidth"] //= 2 else: if state["LocalReadVectorWidth"] == -1: @@ -2861,7 +1778,7 @@ def calSwizzleK(state, tc): if state["MatrixInstBM"] == 1 and state["MIWaveTile"][0] == 1 and state["MIWaveGroup"][0] == 1 and state["ProblemType"]["TLUA"]: state["GlobalReadVectorWidthA"] = 1 else: - reject(state, "GRVWA=-2 is set for skinny MT") + reject(state, printRejectionReason, "GRVWA=-2 is set for skinny MT") elif state["GlobalReadVectorWidthA"] == -1: if state["ProblemType"]["SwizzleTensorA"]: state["GlobalReadVectorWidthA"] = state["MIInputPerThreadA"] * calSwizzleK(state, "A") @@ -2882,7 +1799,7 @@ def calSwizzleK(state, tc): if state["MatrixInstBN"] == 1 and state["MIWaveTile"][1] == 1 and state["MIWaveGroup"][1] == 1 and state["ProblemType"]["TLUB"]: state["GlobalReadVectorWidthB"] = 1 else: - reject(state, "GRVWB=-2 is set for skinny MT") + reject(state, printRejectionReason, "GRVWB=-2 is set for skinny MT") elif state["GlobalReadVectorWidthB"] == -1: if state["ProblemType"]["SwizzleTensorB"]: state["GlobalReadVectorWidthB"] = state["MIInputPerThreadB"] * calSwizzleK(state, "B") @@ -2899,34 +1816,34 @@ def calSwizzleK(state, tc): for tc in ("A", "B",): if state["ProblemType"][f"SwizzleTensor{tc}"]: if not state["EnableMatrixInstruction"]: - reject(state, f"Tensor {tc} swizzling supports MI only") + reject(state, printRejectionReason, f"Tensor {tc} swizzling supports MI only") # Print rejection reason instead of force set # 16 means bytes of buffer_load_dwordx4 SwizzlePackK = calSwizzleK(state, tc) if state[f"GlobalReadVectorWidth{tc}"] != state[f"MIInputPerThread{tc}"] * SwizzlePackK: GRVW_TC = state[f"GlobalReadVectorWidth{tc}"] MIInPerThread = state[f"MIInputPerThread{tc}"] - reject(state, f"SwizzleTensor{tc} doesn't support GRVW{tc} ({GRVW_TC}) != MIInputPerThread{tc} ({MIInPerThread}) * {SwizzlePackK}") + reject(state, printRejectionReason, f"SwizzleTensor{tc} doesn't support GRVW{tc} ({GRVW_TC}) != MIInputPerThread{tc} ({MIInPerThread}) * {SwizzlePackK}") # TODO- increasing VW might have better perf. But it'll change the swizzling pattern. if state[f"VectorWidth{tc}"] != 1: VW_TC = state[f"VectorWidth{tc}"] - reject(state, f"SwizzleTensor{tc} requires VectorWidth{tc} ({VW_TC}) == 1") + reject(state, printRejectionReason, f"SwizzleTensor{tc} requires VectorWidth{tc} ({VW_TC}) == 1") if state["ProblemType"]["SwizzleTensorA"]: if not state["DirectToVgprA"]: - reject(state, f"Tensor A swizzling requires DirectToVgprA") + reject(state, printRejectionReason, f"Tensor A swizzling requires DirectToVgprA") if not state["ProblemType"]["TransposeA"]: - reject(state, f"Tensor A swizzling supports TN or TT only") + reject(state, printRejectionReason, f"Tensor A swizzling supports TN or TT only") if state["ProblemType"]["SwizzleTensorB"]: if not state["DirectToVgprB"]: - reject(state, f"Tensor B swizzling requires DirectToVgprB") + reject(state, printRejectionReason, f"Tensor B swizzling requires DirectToVgprB") if state["ProblemType"]["TransposeB"]: - reject(state, f"Tensor B swizzling supports TN or NN only") + reject(state, printRejectionReason, f"Tensor B swizzling supports TN or NN only") # TODO- NN fails validation due to DTVB + Tail-Loop is not working correctly if not (state["ProblemType"]["TransposeA"] and not state["ProblemType"]["TransposeB"]): - reject(state, f"Tensor B swizzling supports TN only") + reject(state, printRejectionReason, f"Tensor B swizzling supports TN only") # Force GRVW the same when UnrollLoopSwapGlobalReadOrder = 1. if genGRVWA and state["UnrollLoopSwapGlobalReadOrder"] == 1: @@ -2936,15 +1853,15 @@ def calSwizzleK(state, tc): # reject - VW too big if (state["VectorWidthA"] * state["ProblemType"]["DataType"].numBytes()) > 16: - reject(state, "VWA * DataType.numBytes() > 16") + reject(state, printRejectionReason, "VWA * DataType.numBytes() > 16") if (state["VectorWidthB"] * state["ProblemType"]["DataType"].numBytes()) > 16: - reject(state, "VWB * DataType.numBytes() > 16") + reject(state, printRejectionReason, "VWB * DataType.numBytes() > 16") # reject - GRVW too big if (state["GlobalReadVectorWidthA"] * state["ProblemType"]["DataTypeA"].numBytes()) > 16: - reject(state, "GRVWA * DataTypeA.numBytes() > 16") + reject(state, printRejectionReason, "GRVWA * DataTypeA.numBytes() > 16") if (state["GlobalReadVectorWidthB"] * state["ProblemType"]["DataTypeB"].numBytes()) > 16: - reject(state, "GRVWB * DataTypeB.numBytes() > 16") + reject(state, printRejectionReason, "GRVWB * DataTypeB.numBytes() > 16") ######################################## # Search DepthU @@ -2992,10 +1909,10 @@ def calSwizzleK(state, tc): totalElementsM = totalElementsCoalescedM * totalElementsPerpM tva = totalElementsA // state["GlobalReadVectorWidthA"] - if not Solution.setGlobalReadVectorWidth(state, "A", tva, state["GlobalReadVectorWidthA"]): + if not Solution.setGlobalReadVectorWidth(state, "A", tva, state["GlobalReadVectorWidthA"], printRejectionReason): validDepthU = False tvb = totalElementsB // state["GlobalReadVectorWidthB"] - if not Solution.setGlobalReadVectorWidth(state, "B", tvb, state["GlobalReadVectorWidthB"]): + if not Solution.setGlobalReadVectorWidth(state, "B", tvb, state["GlobalReadVectorWidthB"], printRejectionReason): validDepthU = False if state["EnableMatrixInstruction"] and state["GlobalReadVectorWidthA"]: @@ -3014,7 +1931,7 @@ def calSwizzleK(state, tc): # reduce GLVA if GLVA larger than MIOVW if state["GlobalReadVectorWidthA"] > glvwAlimit: tva = totalElementsA // glvwAlimit - if not Solution.setGlobalReadVectorWidth(state, "A", tva, glvwAlimit): + if not Solution.setGlobalReadVectorWidth(state, "A", tva, glvwAlimit, printRejectionReason): validDepthU = False if state["EnableMatrixInstruction"] and state["GlobalReadVectorWidthB"]: @@ -3032,14 +1949,14 @@ def calSwizzleK(state, tc): # reduce GLVB if GLVB larger than MIOVW if state["GlobalReadVectorWidthB"] > glvwBlimit: tvb = totalElementsB // glvwBlimit - if not Solution.setGlobalReadVectorWidth(state, "B", tvb, glvwBlimit): + if not Solution.setGlobalReadVectorWidth(state, "B", tvb, glvwBlimit, printRejectionReason): validDepthU = False if validDepthU and state["KernelLanguage"] == "Assembly": - if globalParameters["ArchCaps"][globalParameters["CurrentISA"]]["HasEccHalf"]: + if isaInfoMap[state["ISA"]].archCaps["HasEccHalf"]: if state["ProblemType"]["DataType"].numRegisters() == 0.5 and (not state["ProblemType"]["HighPrecisionAccumulate"]): if state["GlobalReadVectorWidthA"] == 1 or state["GlobalReadVectorWidthB"] == 1: - reject(state, "HalfEcc requires HPA if glvw = 1") + reject(state, printRejectionReason, "HalfEcc requires HPA if glvw = 1") break if state["ProblemType"]["Sparse"] and not state["DirectToVgprSparseMetadata"]: @@ -3049,19 +1966,19 @@ def calSwizzleK(state, tc): grvw = state["GlobalReadVectorWidthB"] // 4 vw = state["VectorWidthB"] // 4 if state["GlobalReadVectorWidthB"] % 4 != 0: - reject(state, "Sparse B requires GRVWB %% 4 == 0, current GRVWB is %u"%state["GlobalReadVectorWidthB"]) + reject(state, printRejectionReason, "Sparse B requires GRVWB %% 4 == 0, current GRVWB is %u"%state["GlobalReadVectorWidthB"]) break else: grvw = state["GlobalReadVectorWidthA"] // 4 vw = state["VectorWidthA"] // 4 if state["GlobalReadVectorWidthA"] % 4 != 0: - reject(state, "Sparse A requires GRVWA %% 4 == 0, current GRVWA is %u"%state["GlobalReadVectorWidthA"]) + reject(state, printRejectionReason, "Sparse A requires GRVWA %% 4 == 0, current GRVWA is %u"%state["GlobalReadVectorWidthA"]) break tvm = totalElementsM // grvw - if not Solution.setGlobalReadVectorWidth(state, "Metadata", tvm, grvw): + if not Solution.setGlobalReadVectorWidth(state, "Metadata", tvm, grvw, printRejectionReason): validDepthU = False if state["EnableMatrixInstruction"] and state["GlobalReadVectorWidthMetadata"]: @@ -3084,12 +2001,12 @@ def calSwizzleK(state, tc): # reduce GLVMetadata if GLVMetadata larger than MIOVW if state["GlobalReadVectorWidthMetadata"] > glvwMlimit: tvm = totalElementsM // glvwMlimit - if not Solution.setGlobalReadVectorWidth(state, "Metadata", tvm, glvwMlimit): + if not Solution.setGlobalReadVectorWidth(state, "Metadata", tvm, glvwMlimit, printRejectionReason): validDepthU = False if state["ProblemType"]["Sparse"] and state["DirectToVgprSparseMetadata"]: if state["VectorWidthA"] > 1 or state["VectorWidthB"] > 1 : - reject(state, "Not implement DTVSM with VW>1") + reject(state, printRejectionReason, "Not implement DTVSM with VW>1") break # Now convert elements to vectors based on GlobalReadVectorWidth @@ -3118,7 +2035,7 @@ def calSwizzleK(state, tc): # this depthU not valid else: - reject(state, "No valid DepthU found") + reject(state, printRejectionReason, "No valid DepthU found") state["ValidDepthU"] = False break ######################################## @@ -3167,25 +2084,25 @@ def calSwizzleK(state, tc): if state["EnableMatrixInstruction"]: if state["SourceSwap"]: if ((state["VectorWidthA"] % state["StoreVectorWidth"]) != 0): - reject(state, "MFMA SourceSwap mode doesn't support vwA(%u) with svw(%u)" % (state["VectorWidthA"], state["StoreVectorWidth"])) + reject(state, printRejectionReason, "MFMA SourceSwap mode doesn't support vwA(%u) with svw(%u)" % (state["VectorWidthA"], state["StoreVectorWidth"])) return else: if (((state["VectorWidthA"] * state["MIOutputVectorWidth"]) % state["StoreVectorWidth"]) != 0): - reject(state, "MFMA non-SourceSwap mode doesn't support miovw(%u) with svw(%u)" % (state["VectorWidthA"]*state["MIOutputVectorWidth"], state["StoreVectorWidth"])) + reject(state, printRejectionReason, "MFMA non-SourceSwap mode doesn't support miovw(%u) with svw(%u)" % (state["VectorWidthA"]*state["MIOutputVectorWidth"], state["StoreVectorWidth"])) return # LocalSplitU too large? numElementsPerWorkGroup = state["MacroTile0"]*state["MacroTile1"] if numElementsPerWorkGroup < state["NumThreads"]: - reject(state, "NumElementsPerWorkGroup %u < NumThreads %u; reduce LocalSplitU" \ + reject(state, printRejectionReason, "NumElementsPerWorkGroup %u < NumThreads %u; reduce LocalSplitU" \ % (numElementsPerWorkGroup, state["NumThreads"])) return state["NumElementsPerThread"] = numElementsPerWorkGroup // state["NumThreads"] state["GlobalWriteVectorWidth"] = min(state["VectorWidthA"], state["NumElementsPerThread"] ) if state["NumElementsPerThread"] % state["GlobalWriteVectorWidth"] != 0: - reject(state, "LSU NumElementsPerThread %u not divisible into GWVW %u" \ + reject(state, printRejectionReason, "LSU NumElementsPerThread %u not divisible into GWVW %u" \ % (state["NumElementsPerThread"], state["GlobalWriteVectorWidth"])) return state["NumGlobalWriteVectorsPerThread"] = state["NumElementsPerThread"] \ @@ -3195,21 +2112,21 @@ def calSwizzleK(state, tc): # LocalSplitU but can't NumThreads%MacroTile doesn't support sideways store if state["LocalSplitU"] > 1: if not state["SourceSwap"] and state["StoreVectorWidth"] > state["VectorWidthA"]: - reject(state, "LSU and non-SourceSwap doesn't support StoreVectorWidth(%u)>VWA(%u)." \ + reject(state, printRejectionReason, "LSU and non-SourceSwap doesn't support StoreVectorWidth(%u)>VWA(%u)." \ % (state["StoreVectorWidth"], state["VectorWidthA"])) return if not (state["ProblemType"]["ComputeDataType"].isSingle() or state["ProblemType"]["ComputeDataType"].isInt32()): - reject(state, "TODO: LSU doesn't support ComputeDataType!=(single or Int32).") + reject(state, printRejectionReason, "TODO: LSU doesn't support ComputeDataType!=(single or Int32).") return if state["StoreRemapVectorWidth"] > 0: - reject(state, "TODO: LSU doesn't support StoreRemapVectorWidth>0.") + reject(state, printRejectionReason, "TODO: LSU doesn't support StoreRemapVectorWidth>0.") return if state["NumThreads"] % state["MacroTile0"] != 0: - reject(state, "LocalSplitU but NumThreads=%u not divisible by MT0=%u for sideways store" \ + reject(state, printRejectionReason, "LocalSplitU but NumThreads=%u not divisible by MT0=%u for sideways store" \ % (state["NumThreads"], state["MacroTile0"])) return if state["MacroTile0"]*state["MacroTile1"] % state["NumThreads"] != 0: - reject(state, "LocalSplitU but MT0*MT1=%u elements doesn't divide into NumThreads=%u" \ + reject(state, printRejectionReason, "LocalSplitU but MT0*MT1=%u elements doesn't divide into NumThreads=%u" \ % (state["MacroTile0"]*state["MacroTile1"], state["NumThreads"])) return @@ -3225,26 +2142,26 @@ def calSwizzleK(state, tc): (state["_GlobalAccumulation"]) ) if not supported: - reject(state, "GlobalSplitU only compatible with single or asm and (half or mixed) precision") + reject(state, printRejectionReason, "GlobalSplitU only compatible with single or asm and (half or mixed) precision") return if state["ProblemType"]["DataType"].isHalf() and state["KernelLanguage"] == "Assembly": if state["GlobalSplitU"] > 1 and (not state["_GlobalAccumulation"]): if state["AssertFree0ElementMultiple"] < 2: - reject(state, "Assembly GSU half requires AF0EM>=2 (for atomics on edge tiles)") + reject(state, printRejectionReason, "Assembly GSU half requires AF0EM>=2 (for atomics on edge tiles)") - if state["EnableMatrixInstruction"] and globalParameters["AsmCaps"][isa]['HasWMMA']: - reject(state, "Half WMMA doesn't support single buffer GSU") + if state["EnableMatrixInstruction"] and isaInfoMap[isa].asmCaps['HasWMMA']: + reject(state, printRejectionReason, "Half WMMA doesn't support single buffer GSU") return if state["ProblemType"]["Sparse"] and not state["DirectToVgprSparseMetadata"]: state["NumLoadsCoalescedMetadata"] = 1 if not Solution.setGlobalLoadTileDimClassic(state, "A", state["NumLoadsA"], \ - totalVectorsCoalescedA, totalElementsPerpA, depthUA): + totalVectorsCoalescedA, totalElementsPerpA, depthUA, printRejectionReason): return if not Solution.setGlobalLoadTileDimClassic(state, "B", state["NumLoadsB"], \ - totalVectorsCoalescedB, totalElementsPerpB, depthUB): + totalVectorsCoalescedB, totalElementsPerpB, depthUB, printRejectionReason): return if state["ProblemType"]["Sparse"] and not state["DirectToVgprSparseMetadata"]: @@ -3263,10 +2180,10 @@ def calSwizzleK(state, tc): if state["ProblemType"]["Sparse"] == 2: GlobalReadVectorWidth = min(state["GlobalReadVectorWidthMetadata"] * state["NumLoadsPerpendicularB"], depthUM) #sum all need read tvm = totalElementsM // GlobalReadVectorWidth - if not Solution.setGlobalReadVectorWidth(state, "Metadata", tvm, GlobalReadVectorWidth): + if not Solution.setGlobalReadVectorWidth(state, "Metadata", tvm, GlobalReadVectorWidth, printRejectionReason): #fallback tvm = totalElementsM // bGlobalReadVectorWidthMetadata - Solution.setGlobalReadVectorWidth(state, "Metadata", tvm, bGlobalReadVectorWidthMetadata) + Solution.setGlobalReadVectorWidth(state, "Metadata", tvm, bGlobalReadVectorWidthMetadata, printRejectionReason) GlobalReadVectorWidthMetadata = state["GlobalReadVectorWidthMetadata"] if GlobalReadVectorWidthMetadata == 0: @@ -3276,10 +2193,10 @@ def calSwizzleK(state, tc): else: GlobalReadVectorWidth = min(state["GlobalReadVectorWidthMetadata"] * state["NumLoadsPerpendicularA"], depthUM) #sum all need read tvm = totalElementsM // GlobalReadVectorWidth - if not Solution.setGlobalReadVectorWidth(state, "Metadata", tvm, GlobalReadVectorWidth): + if not Solution.setGlobalReadVectorWidth(state, "Metadata", tvm, GlobalReadVectorWidth, printRejectionReason): #fallback tvm = totalElementsM // bGlobalReadVectorWidthMetadata - Solution.setGlobalReadVectorWidth(state, "Metadata", tvm, bGlobalReadVectorWidthMetadata) + Solution.setGlobalReadVectorWidth(state, "Metadata", tvm, bGlobalReadVectorWidthMetadata, printRejectionReason) GlobalReadVectorWidthMetadata = state["GlobalReadVectorWidthMetadata"] if GlobalReadVectorWidthMetadata == 0: @@ -3288,21 +2205,21 @@ def calSwizzleK(state, tc): totalVectorsM = totalElementsM // GlobalReadVectorWidthMetadata if not Solution.setGlobalLoadTileDimClassic(state, "Metadata", state["NumLoadsMetadata"], \ - totalVectorsCoalescedM, totalElementsPerpM, depthUM): + totalVectorsCoalescedM, totalElementsPerpM, depthUM, printRejectionReason): return # TODO if (0 and state["LSCA"] % state["GlobalReadVectorWidthA"] != 0): - reject(state, "lsca % grvw != 0") + reject(state, printRejectionReason, "lsca % grvw != 0") return if (0 and state["LSPA"] % state["GlobalReadVectorWidthA"] != 0): - reject(state, "lspa % grvw != 0") + reject(state, printRejectionReason, "lspa % grvw != 0") return if (0 and state["LSCB"] % state["GlobalReadVectorWidthB"] != 0): - reject(state, "lscb % grvw != 0") + reject(state, printRejectionReason, "lscb % grvw != 0") return if (0 and state["LSPB"] % state["GlobalReadVectorWidthB"] != 0): - reject(state, "lspb % grvw != 0") + reject(state, printRejectionReason, "lspb % grvw != 0") return state["LVCA"] = roundupRatio(state["LSCA"] , state["GlobalReadVectorWidthA"]) @@ -3328,7 +2245,7 @@ def calSwizzleK(state, tc): # lds buffer size for A, B if state["KernelLanguage"] == "Source" and \ state["LdsPadA"] != state["LdsPadB"]: - reject(state, "Source KernelLanguage only supports LdsPadA == LdsPadB") + reject(state, printRejectionReason, "Source KernelLanguage only supports LdsPadA == LdsPadB") return # NoTailLoop parameter initialization. @@ -3341,10 +2258,10 @@ def calSwizzleK(state, tc): # Determine if we can load directly-to-Vgpr # need to check after state["LocalReadVectorWidth"] = -1 is resolved if state["DirectToVgprA"]: - if not Solution.isDirectToVgprDoable(state, 'A'): + if not Solution.isDirectToVgprDoable(state, 'A', printRejectionReason, isaInfoMap): return # rejected if state["DirectToVgprB"]: - if not Solution.isDirectToVgprDoable(state, 'B'): + if not Solution.isDirectToVgprDoable(state, 'B', printRejectionReason, isaInfoMap): return # rejected ######################################## @@ -3373,23 +2290,23 @@ def calSwizzleK(state, tc): if state["LdsBlockSizePerPadA"]: if state["UnrollMajorLDSA"]: if state["LdsBlockSizePerPadA"] % (state["_DepthUA"] * state["ProblemType"]["DataTypeA"].numBytes()) != 0: - reject(state, "reject: LdsBlockSizePerPadA %u %% depthU %u x bpeA != 0" % (state["LdsBlockSizePerPadA"],state["_DepthUA"])) + reject(state, printRejectionReason, "reject: LdsBlockSizePerPadA %u %% depthU %u x bpeA != 0" % (state["LdsBlockSizePerPadA"],state["_DepthUA"])) if (state["LdsBlockSizePerPadA"] // (state["_DepthUA"] * state["ProblemType"]["DataType"].numBytes())) % state["LSPA"] != 0 and \ state["LSPA"] % (state["LdsBlockSizePerPadA"] // (state["_DepthUA"] * state["ProblemType"]["DataType"].numBytes())) != 0: - reject(state, "can't pad by addrVgpr or instOffset") + reject(state, printRejectionReason, "can't pad by addrVgpr or instOffset") if state["LdsBlockSizePerPadB"]: if state["UnrollMajorLDSB"]: if state["LdsBlockSizePerPadB"] % state["_DepthUB"] * state["ProblemType"]["DataTypeB"].numBytes() != 0: - reject(state, "reject: LdsBlockSizePerPadB %u %% depthU %u x bpeB != 0" % (state["LdsBlockSizePerPadB"],state["_DepthUB"])) + reject(state, printRejectionReason, "reject: LdsBlockSizePerPadB %u %% depthU %u x bpeB != 0" % (state["LdsBlockSizePerPadB"],state["_DepthUB"])) if (state["LdsBlockSizePerPadB"] // (state["_DepthUB"] * state["ProblemType"]["DataType"].numBytes())) % state["LSPB"] != 0 and \ state["LSPB"] % (state["LdsBlockSizePerPadB"] // (state["_DepthUB"] * state["ProblemType"]["DataType"].numBytes())) != 0: - reject(state, "can't pad by addrVgpr or instOffset") + reject(state, printRejectionReason, "can't pad by addrVgpr or instOffset") else: if state["UnrollMajorLDSA"] or state["UnrollMajorLDSB"]: - reject(state, "didn't support UnrollMajorLDS in VALU mode yet") + reject(state, printRejectionReason, "didn't support UnrollMajorLDS in VALU mode yet") if state["LdsBlockSizePerPadA"] != 0 or state["LdsBlockSizePerPadB"] != 0: - reject(state, "didn't support LdsBlockSizePerPad in VALU mode yet") + reject(state, printRejectionReason, "didn't support LdsBlockSizePerPad in VALU mode yet") def checkLdsBlockSizePerPad(tc): """ @@ -3477,7 +2394,7 @@ def findValidWriteBlockWidth(nwcv, bpe, bpr): blockWidth = bw break if blockWidth == 0: - reject(state, "invalid local write block width") + reject(state, printRejectionReason, "invalid local write block width") return blockWidth @@ -3539,7 +2456,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): printWarning("Padded address is inconisstent, set LdsBlockSizePerPad%s=0."%tc) state["LdsBlockSizePerPad%s"%tc] = 0 else: - reject(state, "%s's padded address is inconisstent"%tc) + reject(state, printRejectionReason, "%s's padded address is inconisstent"%tc) if(not (state["CustomKernelName"] and state["CustomKernelName"] != "")): #don't check the custom kernel. checkLdsBlockSizePerPad("A") @@ -3557,12 +2474,12 @@ def subCheckLdsBlockSizePerPad(tc, idx): # LDS (load size coalesced) * LSPA must load some multiple of 256 bytes. # No longer support loadX2/loadx4 . if state["DirectToLds"]: - if (not state["DirectToVgprA"]) and Solution.isDirectToLdsDoable(state, 'A'): + if (not state["DirectToVgprA"]) and Solution.isDirectToLdsDoable(state, 'A', printRejectionReason): state["DirectToLdsA"] = True state["LocalWriteUseSgprA"] = True #print("DirectToLdsA", state["DirectToLdsA"]) - if (not state["DirectToVgprB"]) and Solution.isDirectToLdsDoable(state, 'B'): + if (not state["DirectToVgprB"]) and Solution.isDirectToLdsDoable(state, 'B', printRejectionReason): state["DirectToLdsB"] = True state["LocalWriteUseSgprB"] = True #print("DirectToLdsB", state["DirectToLdsB"]) @@ -3576,10 +2493,10 @@ def subCheckLdsBlockSizePerPad(tc, idx): # Re-check DTV + WaveGroup after DTL is confirmed if state["DirectToLds"]: if state["DirectToVgprA"] and state['MIWaveGroup'][1] > 1: - reject(state, "DirectToLds + (DirectToVgprA + WaveGroups along N-Dim) is not supported yet") + reject(state, printRejectionReason, "DirectToLds + (DirectToVgprA + WaveGroups along N-Dim) is not supported yet") return False if state["DirectToVgprB"] and state['MIWaveGroup'][0] > 1: - reject(state, "DirectToLds + (DirectToVgprB + WaveGroups along M-Dim) is not supported yet") + reject(state, printRejectionReason, "DirectToLds + (DirectToVgprB + WaveGroups along M-Dim) is not supported yet") return False # set NoLdsWriteCode if (DirectToVgpr or DirectToLds)A+B is enabled @@ -3588,7 +2505,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): state["NoLdsWriteCode"] = True # calculate ldsPad - state["LdsPadA"], state["LdsPadB"], state["LdsPadMetadata"] = calcLdsPad(state["LocalReadVectorWidth"]) + state["LdsPadA"], state["LdsPadB"], state["LdsPadMetadata"] = calcLdsPad(state["LocalReadVectorWidth"], isaInfoMap) if state["GlobalReadVectorWidthA"] * state["ProblemType"]["DataType"].numBytes() == 32 and state["LdsPadA"] == 16 // state["ProblemType"]["DataType"].numBytes(): if auto_LdsBlockSizePerPadA_for_mix: @@ -3601,7 +2518,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): assert(state["LdsPadB"] >= 0) if (state["UnrollMajorLDSA"] or state["UnrollMajorLDSB"]) and (not state["EnableMatrixInstruction"]): - reject(state, "UnrollMajorLDS Supports only in EnableMatrixInstruction=1") + reject(state, printRejectionReason, "UnrollMajorLDS Supports only in EnableMatrixInstruction=1") ldsNumBytesA, ldsNumBytesAlignedA, ldsNumBytesB, ldsNumBytesAlignedB, ldsNumBytesMetadata, ldsNumBytesAlignedMetadata = calcLdsNumBytes(state["LdsPadA"], state["LdsBlockSizePerPadA"], state["LdsPadB"], state["LdsBlockSizePerPadB"]) @@ -3631,13 +2548,13 @@ def subCheckLdsBlockSizePerPad(tc, idx): # if User want to control the LDS usage, we may open this para in the future ldsNumBytesReduction = state["LocalSplitU"] * state["MacroTile0"] * state["MacroTile1"] * state["ProblemType"]["ComputeDataType"].numBytes() if state["LocalSplitU"] > 1 else 0 state["LocalSplitUReuseLDS"] = 1 - if ldsNumBytesReduction > globalParameters["MaxLDS"]: - state["LocalSplitUReuseLDS"] = math.ceil(ldsNumBytesReduction / globalParameters["MaxLDS"]) + if ldsNumBytesReduction > depthUConfig.maxLDS: + state["LocalSplitUReuseLDS"] = math.ceil(ldsNumBytesReduction / depthUConfig.maxLDS) # reserve all the LDS to LSU. - ldsNumBytesReduction = globalParameters["MaxLDS"] + ldsNumBytesReduction = depthUConfig.maxLDS # lds max occupancy - ldsSizeOccupancy = globalParameters["DeviceLDS"] // state["MaxOccupancy"] + ldsSizeOccupancy = depthUConfig.deviceLDS // state["MaxOccupancy"] ldsNumBytesOccupancy = ldsSizeOccupancy #print("LdsOffsetB", state["LdsOffsetB"]) @@ -3649,22 +2566,22 @@ def subCheckLdsBlockSizePerPad(tc, idx): if state["EnableMatrixInstruction"]: if state["DirectToLds"] and state["1LDSBuffer"]: - reject(state, "1LDSBuffer must be 0 for directToLds") + reject(state, printRejectionReason, "1LDSBuffer must be 0 for directToLds") if state["1LDSBuffer"] == -1: if ldsNumBytesAB <= max(ldsSizeOccupancy,32768) or \ (state["ProblemType"]["ComputeDataType"].numBytes() * state["MacroTile0"] * state["MacroTile1"] > 32768*4 and \ - not (ldsNumBytesAB > globalParameters["DeviceLDS"])): + not (ldsNumBytesAB > depthUConfig.deviceLDS)): state["1LDSBuffer"] = 0 else: state["1LDSBuffer"] = 1 if state["1LDSBuffer"]: if not state["PrefetchGlobalRead"]: - reject(state, "PGR=0 already use 1 LDS buffer only") + reject(state, printRejectionReason, "PGR=0 already use 1 LDS buffer only") # Should be able to support as long as NO scheduleLocalWrite if (not state["ScheduleIterAlg"] == 2) and (not state["ScheduleIterAlg"] == 3) and (state["ScheduleLocalWrite"]): - reject(state, "1LDSBuffer only support SIA2 or SIA3, or SIA1 without SLW") + reject(state, printRejectionReason, "1LDSBuffer only support SIA2 or SIA3, or SIA1 without SLW") state["LdsOffsetB"] = ldsNumBytesAlignedA state["LdsOffsetMetadata"] = state["LdsOffsetB"] + ldsNumBytesAlignedB ldsNumBytesAB = ldsNumBytesAlignedA + ldsNumBytesAlignedB + ldsNumBytesMetadata @@ -3694,7 +2611,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): ldsNumElementsRemapC = max(ldsNumElementsRemapC, ldsNumElementsRemapC * (computeBytes / state["ProblemType"]["DestDataType"].numBytes())) ldsSize = ldsNumElementsRemapC * state["ProblemType"]["DestDataType"].numBytes() if not math.log(state["MacroTile0"],2).is_integer() or \ - ldsSize > globalParameters["MaxLDS"] or \ + ldsSize > depthUConfig.maxLDS or \ state["SourceSwap"] or \ (state["GlobalSplitU"] > 1) and (state["_GlobalAccumulation"] != 'MultipleBuffer') or \ state["MatrixInstBN"] > 1 and state["MatrixInstN"] == 4 : @@ -3706,9 +2623,9 @@ def subCheckLdsBlockSizePerPad(tc, idx): if not state["SourceSwap"]: if not state["StoreRemapVectorWidth"]: - reject(state, "reject to reduce number of kernels") + reject(state, printRejectionReason, "reject to reduce number of kernels") elif state["VectorWidthA"] > 1: - reject(state, "reject to reduce number of kernels") + reject(state, printRejectionReason, "reject to reduce number of kernels") # GuaranteeNoPartial if state["ProblemType"]["TLUA"]: @@ -3726,73 +2643,73 @@ def subCheckLdsBlockSizePerPad(tc, idx): # SourceSwap if state["StoreRemapVectorWidth"]: if state["SourceSwap"]: - reject(state, "SourceSwap not compatible with StoreRemap") + reject(state, printRejectionReason, "SourceSwap not compatible with StoreRemap") return if state["VectorWidthA"] > 1 or state["VectorWidthB"] > 1: - reject(state, "VW>1 not compatible with StoreRemap") + reject(state, printRejectionReason, "VW>1 not compatible with StoreRemap") return # Sparse problem if state["ProblemType"]["Sparse"]: if state["PrefetchGlobalRead"] and not state["ExpandPointerSwap"]: - reject(state, "Sparse A kernel only support PGR with EPS=1.") + reject(state, printRejectionReason, "Sparse A kernel only support PGR with EPS=1.") return if state["EnableMatrixInstruction"] and state["MIArchVgpr"]: - reject(state, "Sparse A kernel does not support MIArchVgpr yet.") + reject(state, printRejectionReason, "Sparse A kernel does not support MIArchVgpr yet.") return # Not Support Feature if state["ProblemType"]["Sparse"] == 1 and state["SourceSwap"] : - reject(state, "Sparse A kernel cannot support SourceSwap.") + reject(state, printRejectionReason, "Sparse A kernel cannot support SourceSwap.") return else: if state["ProblemType"]["Sparse"] == 2 and not state["SourceSwap"]: - reject(state, "Sparse B kernel must enable SourceSwap.") + reject(state, printRejectionReason, "Sparse B kernel must enable SourceSwap.") return state["AssertSummationElementMultiple"] = 8 # check if need to use lds init Acc vgprs state["LdsInitCVgprs"] = False - if globalParameters["ArchCaps"][isa]["HasAccCD"] and \ + if isaInfoMap[isa].archCaps["HasAccCD"] and \ state["EnableMatrixInstruction"] and state["StorePriorityOpt"] and \ state["ProblemType"]["DataType"].isDouble(): state["LdsInitCVgprs"] = True # force MIArchVgpr when using WMMA - if state["EnableMatrixInstruction"] and globalParameters["AsmCaps"][isa]["HasWMMA"]: + if state["EnableMatrixInstruction"] and isaInfoMap[isa].asmCaps["HasWMMA"]: state["MIArchVgpr"] = True if state["MIArchVgpr"]: if not state["EnableMatrixInstruction"]: - reject(state, "MIArchVgpr only support for MatrixInstruction") + reject(state, printRejectionReason, "MIArchVgpr only support for MatrixInstruction") return - if globalParameters["AsmCaps"][isa]["HasMFMA"]: + if isaInfoMap[isa].asmCaps["HasMFMA"]: if not (state["ProblemType"]["ComputeDataType"].isDouble() or \ state["ProblemType"]["ComputeDataType"].isSingle() or \ (state["ProblemType"]["ComputeDataType"].isHalf() and state["ProblemType"]["HighPrecisionAccumulate"]) or \ state["ProblemType"]["ComputeDataType"].isInt32() or \ state["ProblemType"]["ComputeDataType"].isComplex()): - reject(state, "MIArchVgpr now only support fp64, fp64c, fp32, fp32c, fp16, int8 MatrixInstruction.") + reject(state, printRejectionReason, "MIArchVgpr now only support fp64, fp64c, fp32, fp32c, fp16, int8 MatrixInstruction.") return #check not support cases and calculate lds resources ldsNumBytesRemapC = 0 if state["StoreRemapVectorWidth"]: if not state["EnableMatrixInstruction"]: - reject(state, "storeRemap only support MatrixInstruction kernel") + reject(state, printRejectionReason, "storeRemap only support MatrixInstruction kernel") return if ((state["GlobalSplitU"] > 1) and (state["_GlobalAccumulation"] != 'MultipleBuffer' or state["_GlobalAccumulation"] == 'MultipleBufferSingleKernel')) or \ (state["GlobalSplitU"] == 1 and state["_GlobalAccumulation"] == 'SingleBuffer'): - reject(state, "storeRemap doesn't support GlobalSplitU yet, except GSU algorithm 2") + reject(state, printRejectionReason, "storeRemap doesn't support GlobalSplitU yet, except GSU algorithm 2") return if packedC0 or packedC1: - reject(state, "storeRemap doesn't support packedC0 and packedC1 yet") + reject(state, printRejectionReason, "storeRemap doesn't support packedC0 and packedC1 yet") return if state["MatrixInstBN"] > 1 and state["MatrixInstN"] == 4: - reject(state, "storeRemap doesn't support MI4x4 multi blocks in N direction yet") + reject(state, printRejectionReason, "storeRemap doesn't support MI4x4 multi blocks in N direction yet") return if not math.log(state["MacroTile0"],2).is_integer(): - reject(state, "storeRemap only supports power-of-2 MT0") + reject(state, printRejectionReason, "storeRemap only supports power-of-2 MT0") # TODO - this return should be here, but this is a hotfix, # Somehow we have a "Validation Failed" kernel in rocBLAS now (SRVW=4 and MT0=96) and this will stop the whole building process # Actions: 1. Hotfix, comment out this "return" temporarily for that invalidated kernel @@ -3811,15 +2728,15 @@ def subCheckLdsBlockSizePerPad(tc, idx): while srMaxVw < state["StoreRemapVectorWidth"]: state["StoreRemapVectorWidth"] = state["StoreRemapVectorWidth"] // 2 if srMinVw > state["StoreRemapVectorWidth"] or srMaxVw < state["StoreRemapVectorWidth"]: - reject(state, "StoreRemapVectorWidth %u is not allowed for this data type" % state["StoreRemapVectorWidth"]) + reject(state, printRejectionReason, "StoreRemapVectorWidth %u is not allowed for this data type" % state["StoreRemapVectorWidth"]) return if state["StoreRemapVectorWidth"] * state["WavefrontSize"] < state["MacroTile0"]: - reject(state, "storeRemap: Per wave single global write instruction doesn't enough to write one M column." + \ + reject(state, printRejectionReason, "storeRemap: Per wave single global write instruction doesn't enough to write one M column." + \ " Please use larger StoreRemapVectorWidth.") return if (state["MacroTile0"]*state["MatrixInstN"])//state["MIWaveGroup"][0] < state["StoreRemapVectorWidth"]*state["WavefrontSize"]: - reject(state, "storeRemap: number elements of lds less than per wave per local read elements." + \ + reject(state, printRejectionReason, "storeRemap: number elements of lds less than per wave per local read elements." + \ " Please use smaller StoreRemapVectorWidth.") return ldsRemapPad = max(state["StoreRemapVectorWidth"],state["MIOutputVectorWidth"]) @@ -3848,7 +2765,7 @@ def subCheckLdsBlockSizePerPad(tc, idx): # TODO- Remove this DataType test condition, # Currently we do this test is just because we don't want to affect existing logic in rocBLAS if state["ProblemType"]["DataType"].isInt8(): - reject(state, "LDS usage is bound be StoreRemap, thus 1LDSBuffer wouldn't have any help. Skip.") + reject(state, printRejectionReason, "LDS usage is bound be StoreRemap, thus 1LDSBuffer wouldn't have any help. Skip.") return ldsNumBytes = max(ldsNumBytes, ldsNumBytesRemapC) @@ -3888,7 +2805,7 @@ def calcEpilogueTurns(factorDims: List) -> int: maxTurn = calcEpilogueTurns([0, 1]) vecDT.bias(0).turn = maxTurn vecDT.bias(1).turn = maxTurn - + # Calc LDS for SAV maxTurn = 0 if savDim == 1: @@ -3944,8 +2861,8 @@ def calcEpilogueTurns(factorDims: List) -> int: state["LdsNumBytes"] = ldsNumBytes ldsSize = ldsNumBytes - if ldsSize > globalParameters["MaxLDS"]: - reject(state, "Kernel Uses %u > %u bytes of LDS" % ( ldsSize, globalParameters["MaxLDS"])) + if ldsSize > depthUConfig.maxLDS: + reject(state, printRejectionReason, "Kernel Uses %u > %u bytes of LDS" % ( ldsSize, depthUConfig.maxLDS)) state["ValidDepthU"] = False return @@ -3955,7 +2872,7 @@ def calcEpilogueTurns(factorDims: List) -> int: if state["LoopUnroll"] * state["LocalSplitU"] != state["DepthU"]: state["Valid"] = False if state["KernelLanguage"] != "Assembly" and state["InnerUnroll"] != 1: - reject(state, "InnerUnroll only supported on assembly") + reject(state, printRejectionReason, "InnerUnroll only supported on assembly") state["LoopUnroll"] //= state["InnerUnroll"] if 0: @@ -3972,7 +2889,7 @@ def calcEpilogueTurns(factorDims: List) -> int: state["LoopIters"] //= state["MatrixInstK"] if state["LoopIters"] < 1: - reject(state, "LoopIters need to greater than 0") + reject(state, printRejectionReason, "LoopIters need to greater than 0") return # Since we use PLR >= LoopIters for allocating numberOfIters vgprBuffer for a while @@ -3981,7 +2898,7 @@ def calcEpilogueTurns(factorDims: List) -> int: # 1 or 2 Byte input + DTVA or DTVB case, does not work with PLR=0. Reject it here. if state["ProblemType"]["DataType"].numBytes() < 4 and \ (state["ProblemType"]["TLUA"] and state["DirectToVgprA"] or state["ProblemType"]["TLUB"] and state["DirectToVgprB"]): - reject(state, "DirectToVgpr does not work with 1 or 2 Byte input + TLU + PrefetchLocalRead(%u) >= LoopIters(%u)"%(state["PrefetchLocalRead"], state["LoopIters"])) + reject(state, printRejectionReason, "DirectToVgpr does not work with 1 or 2 Byte input + TLU + PrefetchLocalRead(%u) >= LoopIters(%u)"%(state["PrefetchLocalRead"], state["LoopIters"])) return state["ClusterLocalRead"] = 0 state["PrefetchLocalRead"] = 0 @@ -3993,31 +2910,31 @@ def calcEpilogueTurns(factorDims: List) -> int: # Multiple = WLR-size / input-size = how many iters could be covered by one WLR ? wlrMultiple = state["LocalReadVectorWidth"]//state["MIInputPerThread"] if wlrMultiple == 0: - reject(state, "LocalReadVectorWidth %u is less than MIInput" % (state["LocalReadVectorWidth"])) + reject(state, printRejectionReason, "LocalReadVectorWidth %u is less than MIInput" % (state["LocalReadVectorWidth"])) return # for example, if the original ds_read is b32... # 1. if LoopIters = 5 (b32 x 5 times), WLR-Multiple = 2 (b64), then we can fit the WLR # 2. if LoopIters = 2 (b32 x 2 times), WLR-Multiple = 4 (b128), this is not allowed # 3. if LoopIters = 2 (b32 x 2 times), WLR-Multiple = 2 (b64), this is allowed if state["LoopIters"] % wlrMultiple != 0: - reject(state, "LocalReadVectorWidth %u cannot be distributed evenly, LoopIters %u should be divisible by WLR-Multiple %u" \ + reject(state, printRejectionReason, "LocalReadVectorWidth %u cannot be distributed evenly, LoopIters %u should be divisible by WLR-Multiple %u" \ % (state["LocalReadVectorWidth"], state["LoopIters"], wlrMultiple)) if state["LoopIters"] - (state["PrefetchLocalRead"] * wlrMultiple) < 0 : - reject(state, "with PrefetchLocalRead %u LoopIters %u LocalReadVectorWidth %u, not enough LoopIters to prefetch %ux%u iterations, " \ + reject(state, printRejectionReason, "with PrefetchLocalRead %u LoopIters %u LocalReadVectorWidth %u, not enough LoopIters to prefetch %ux%u iterations, " \ % (state["PrefetchLocalRead"],state["LoopIters"],state["LocalReadVectorWidth"], state["PrefetchLocalRead"] , wlrMultiple) ) # # reject conditions with lower performance # if state["ScheduleIterAlg"] == 2 and \ # (state["ExpandPointerSwap"] != 1 or state["LoopIters"] != 1 or state["ScheduleGlobalRead"] != 1): - # reject(state, "ScheduleIterAlg 2 only work with EPS1_SGR1, LoopIter=1") + # reject(state, printRejectionReason, "ScheduleIterAlg 2 only work with EPS1_SGR1, LoopIter=1") if state["TransposeLDS"] == 1: if not state["EnableMatrixInstruction"]: - reject(state, "TransposeLds Supports only in MatrixInstruction=1") + reject(state, printRejectionReason, "TransposeLds Supports only in MatrixInstruction=1") if state["ProblemType"]["TLUA"] and state["ProblemType"]["TLUB"]: # TODO: Now in rocBLAS, lot of logic yamls are Type=NT and TLDS=1? Why aren't they rejected and how to get rid of them? - reject(state, "TransposeLds requires TLUA=0 or TLUB=0") + reject(state, printRejectionReason, "TransposeLds requires TLUA=0 or TLUB=0") if state["EnableMatrixInstruction"]: # enable widerLocalRead if state["LocalReadVectorWidth"] > state["MIInputPerThread"]: @@ -4027,10 +2944,10 @@ def calcEpilogueTurns(factorDims: List) -> int: if not (state["PrefetchLocalRead"] >= state["LoopIters"] and state["InnerUnroll"] == 1) and \ not state["ClusterLocalRead"] and \ not state["InnerUnroll"] >= state["LocalReadVectorWidth"] // state["MIInputPerThread"]: - reject(state, "wider localRead only support ClusterLocalRead or (InnerUnroll > WiderLocalReadxN)") + reject(state, printRejectionReason, "wider localRead only support ClusterLocalRead or (InnerUnroll > WiderLocalReadxN)") if state["GlobalReadPerMfma"] > 1 and state["PrefetchGlobalRead"] == 2: - reject(state, "GlobalReadPerMfma need to be 1 if PGR2") + reject(state, printRejectionReason, "GlobalReadPerMfma need to be 1 if PGR2") if state["UseInstOffsetForGRO"] == -1: state["UseInstOffsetForGRO"] = 1 if state["DirectToLds"] else 0 @@ -4044,7 +2961,7 @@ def calcEpilogueTurns(factorDims: List) -> int: numVgprG2LB = roundUp((state["NumLoadsCoalescedB"] * state["NumLoadsPerpendicularB"] * \ state["GlobalReadVectorWidthB"] * bpeAB) / (float)(bpr)) if numVgprG2LA % 2 == 1 or numVgprG2LB % 2 == 1: - reject(state, "G2LA/B vgpr has bubble inside. Cannot use UnrollLoopSwapGlobalReadOrder=1.") + reject(state, printRejectionReason, "G2LA/B vgpr has bubble inside. Cannot use UnrollLoopSwapGlobalReadOrder=1.") if state["GlobalReadVectorWidthA"] != state["GlobalReadVectorWidthB"]: # TODO: Add a configuration to schedule better. state["ULSGRODoubleG2L"] = 1 @@ -4053,11 +2970,11 @@ def calcEpilogueTurns(factorDims: List) -> int: # G2LA/B vgpr index will jump. state["ULSGRODoubleG2L"] = 1 if state["ExpandPointerSwap"] == 1: - reject(state, "ExpandPointerSwap need to be 0 if UnrollLoopSwapGlobalReadOrder") + reject(state, printRejectionReason, "ExpandPointerSwap need to be 0 if UnrollLoopSwapGlobalReadOrder") if state["PrefetchGlobalRead"] != 2: - reject(state, "PrefetchGlobalRead need to be 2 if UnrollLoopSwapGlobalReadOrder") + reject(state, printRejectionReason, "PrefetchGlobalRead need to be 2 if UnrollLoopSwapGlobalReadOrder") if state["ProblemType"]["DataTypeA"].numBytes() != state["ProblemType"]["DataTypeB"].numBytes(): - reject(state, "UnrollLoopSwapGlobalReadOrder doesn't support mixed precision.") + reject(state, printRejectionReason, "UnrollLoopSwapGlobalReadOrder doesn't support mixed precision.") # guard against out of bounds reads # None: don't guard against ou @@ -4086,7 +3003,7 @@ def calcEpilogueTurns(factorDims: List) -> int: if bufferLoad and state["_UseSgprForGRO"] and state["EdgeType"]=="ShiftPtr": if not state["GuaranteeNoPartialA"] or not state["GuaranteeNoPartialB"] or not state["GuaranteeNoPartialMetadata"]: state["_UseSgprForGRO"] = False - #reject(state, "PBC with wide load has insufficient overlap guarantees- try GRVW=1 or adding appropriate Assert*ElementMultiple") + #reject(state, printRejectionReason, "PBC with wide load has insufficient overlap guarantees- try GRVW=1 or adding appropriate Assert*ElementMultiple") @@ -4095,7 +3012,7 @@ def calcEpilogueTurns(factorDims: List) -> int: cont1 = not state["GuaranteeNoPartialB"] cont2 = ((state["MatrixInstN"] % state["GlobalReadVectorWidthB"]) != 0) if cont1 and cont2: - reject(state, "MatrixInstN %u %% GlobalReadVectorWidthB %u must be 0" % \ + reject(state, printRejectionReason, "MatrixInstN %u %% GlobalReadVectorWidthB %u must be 0" % \ (state["MatrixInstN"], state["GlobalReadVectorWidthB"])) # Use SGPR to store an offset from GlobalReadOffsetA+0. @@ -4104,7 +3021,7 @@ def calcEpilogueTurns(factorDims: List) -> int: # individual vector registers doing bounds compares. if state["_UseSgprForGRO"] == 1 and (state["ProblemType"]["SwizzleTensorA"] or state["ProblemType"]["SwizzleTensorB"]): - reject(state, "UseSgprForGRO for Swizzle is not supported") + reject(state, printRejectionReason, "UseSgprForGRO for Swizzle is not supported") if state["_UseSgprForGRO"] == -1: # Don't use SGPR if it looks like we might not have enough - better to leave PBC enabled even if we have to use VGPR @@ -4121,60 +3038,45 @@ def calcEpilogueTurns(factorDims: List) -> int: state["_UseSgprForGRO"] = 1 if packedC0 and not state["GuaranteeNoPartialA"]: - reject(state, "packedC0 requires GuaranteeNoPartialA") + reject(state, printRejectionReason, "packedC0 requires GuaranteeNoPartialA") if packedC1 and not state["GuaranteeNoPartialB"]: - reject(state, "packedC1 requires GuaranteeNoPartialB") + reject(state, printRejectionReason, "packedC1 requires GuaranteeNoPartialB") if packedC0 or packedC1: state["_UseSgprForGRO"] = 0 if state["EdgeType"] != "ShiftPtr": - reject(state, "Packed dims requires EdgeType==ShiftPtr") + reject(state, printRejectionReason, "Packed dims requires EdgeType==ShiftPtr") if state["KernelLanguage"] == "Assembly": if not bufferLoad: - reject(state, "Packed dims for Assembly requires BufferLoad") + reject(state, printRejectionReason, "Packed dims for Assembly requires BufferLoad") if packedC0: # VectorWidth must not span tensor dim if state["KernelLanguage"] == "Source": if state["AssertFree0ElementMultiple"]=VectorWidth (for loads and stores)") + reject(state, printRejectionReason, "packedC0 Source requires AF0EM>=VectorWidth (for loads and stores)") else: if state["AssertFree0ElementMultiple"]=VectorWidth or not VectorStore (for stores)") + reject(state, printRejectionReason, "packedC0 Assembly requires AF0EM>=VectorWidth or not VectorStore (for stores)") state["AssignedDerivedParameters"] = True - # UnrollLoopEfficiencyEnable does not work with f16/bf16/int8x4 - if globalParameters["UnrollLoopEfficiencyEnable"] and (state["ProblemType"]["DataType"].isHalf() or \ - state["ProblemType"]["DataType"].isBFloat16() or state["ProblemType"]["DataType"].isInt8x4()): - reject(state, "UnrollLoopEfficiencyEnable does not support f16/bf16/int8x4") - - # UnrollLoopEfficiencyEnable supports only ThreadTile0,1=[6,4] or [4,6] or [4,4] or [6.6] or [8,4] or [4,8] - if globalParameters["UnrollLoopEfficiencyEnable"] and \ - not ((state["ThreadTile0"] == 6 and state["ThreadTile1"] == 4) or \ - (state["ThreadTile0"] == 4 and state["ThreadTile1"] == 6) or \ - (state["ThreadTile0"] == 4 and state["ThreadTile1"] == 4) or \ - (state["ThreadTile0"] == 6 and state["ThreadTile1"] == 6) or \ - (state["ThreadTile0"] == 8 and state["ThreadTile1"] == 4) or \ - (state["ThreadTile0"] == 4 and state["ThreadTile1"] == 8)): - reject(state, "UnrollLoopEfficiencyEnable does not support ThreadTile0,1 = [%u,%u]"%(state["ThreadTile0"], state["ThreadTile1"])) - # Set E if state["ProblemType"]["UseE"]: if (state["_GlobalAccumulation"] == 'SingleBuffer') and state["GlobalSplitU"] > 1: - reject(state, "GlobalSplitU > 1 only compatible with MultipleBuffer") + reject(state, printRejectionReason, "GlobalSplitU > 1 only compatible with MultipleBuffer") if len(state["PackedC1IndicesX"]) > 1: - reject(state, "Use E does not support len(PackedC1IndicesX) > 1.") + reject(state, printRejectionReason, "Use E does not support len(PackedC1IndicesX) > 1.") if not state["BufferStore"]: - reject(state, "Use E only supports BufferStore due to no suppress no store.") + reject(state, printRejectionReason, "Use E only supports BufferStore due to no suppress no store.") if state["StoreRemapVectorWidth"] and (state["GlobalSplitU"] == 1): - reject(state, "Use E does not support StoreRemapVectorWidth if GSU == 1.") + reject(state, printRejectionReason, "Use E does not support StoreRemapVectorWidth if GSU == 1.") if state["GroupLoadStore"]: - reject(state, "Use E does not support GroupLoadStore.") + reject(state, printRejectionReason, "Use E does not support GroupLoadStore.") # Activation # Function call is set to false if GSU != 1 or Activation is not fused or ActivationType is not All. @@ -4183,24 +3085,24 @@ def calcEpilogueTurns(factorDims: List) -> int: state["ActivationFuncCall"] = False if state["ActivationAlt"]: - reject(state, "Currently does not accept ActivationAlt.") + reject(state, printRejectionReason, "Currently does not accept ActivationAlt.") # Bias reduction if state["ProblemType"]["UseBias"] and state["ProblemType"]["Gradient"]: if (state["_GlobalAccumulation"] == 'SingleBuffer') and state["GlobalSplitU"] > 1: - reject(state, "GlobalSplitU > 1 only compatible with MultipleBuffer for bias reduction") + reject(state, printRejectionReason, "GlobalSplitU > 1 only compatible with MultipleBuffer for bias reduction") if len(state["PackedC1IndicesX"]) > 1: - reject(state, "Bias reduction does not support len(PackedC1IndicesX) > 1.") + reject(state, printRejectionReason, "Bias reduction does not support len(PackedC1IndicesX) > 1.") if not state["BufferStore"]: - reject(state, "Bias reduction only supports BufferStore due to no suppress no store.") + reject(state, printRejectionReason, "Bias reduction only supports BufferStore due to no suppress no store.") if state["StoreRemapVectorWidth"] and (state["GlobalSplitU"] == 1): - reject(state, "Bias reduction does not support StoreRemapVectorWidth if GSU == 1.") + reject(state, printRejectionReason, "Bias reduction does not support StoreRemapVectorWidth if GSU == 1.") if state["GroupLoadStore"]: - reject(state, "Bias reduction does not support GroupLoadStore.") + reject(state, printRejectionReason, "Bias reduction does not support GroupLoadStore.") # Bias and ScaleAlphaVec if state["ProblemType"]["UseBias"] != 0 and state["ProblemType"]["UseScaleAlphaVec"] != 0 and state["ProblemType"]["UseBias"] != state["ProblemType"]["UseScaleAlphaVec"]: - reject(state, "When both UseBias and UseScaleAlphaVec are enabled then UseBias and UseScaleAlphaVec must have same settings.") + reject(state, printRejectionReason, "When both UseBias and UseScaleAlphaVec are enabled then UseBias and UseScaleAlphaVec must have same settings.") # ScaleAB or ScaleABVec if state["ProblemType"]["DataTypeA"] != state["ProblemType"]["DataTypeB"] and \ @@ -4216,243 +3118,29 @@ def calcEpilogueTurns(factorDims: List) -> int: # if state["GlobalSplitU"] > 1: # if state["ProblemType"]["SupportUserArgs"] and state["_GlobalAccumulation"] != 'MultipleBufferSingleKernel': - # reject(state, "Currently SupportUserArgs does not support GSU > 1.") + # reject(state, printRejectionReason, "Currently SupportUserArgs does not support GSU > 1.") if state["_GlobalAccumulation"] == 'MultipleBufferSingleKernel': - if (state["NumElementsPerBatchStore"] == 1): - reject(state, "too many store at MultipleBufferSingleKernel direct reject") + if state["NumElementsPerBatchStore"] == 1: + reject(state, printRejectionReason, "too many store at MultipleBufferSingleKernel direct reject") if state["ProblemType"]["UseScaleCD"]: - reject(state, "MultipleBufferSingleKernel not support UseScaleCD yet") + reject(state, printRejectionReason, "MultipleBufferSingleKernel not support UseScaleCD yet") if state["ProblemType"]["UseE"]: - reject(state, "MultipleBufferSingleKernel not support UseE yet") + reject(state, printRejectionReason, "MultipleBufferSingleKernel not support UseE yet") if state["ProblemType"]["BiasSrc"] != "D": - reject(state, "MultipleBufferSingleKernel not support BiasSrc not D yet") + reject(state, printRejectionReason, "MultipleBufferSingleKernel not support BiasSrc not D yet") if state["ProblemType"]["DataType"].isDouble(): - reject(state, "MultipleBufferSingleKernel not support " + str(state["ProblemType"]["DataType"]) + " yet") + reject(state, printRejectionReason, "MultipleBufferSingleKernel not support " + str(state["ProblemType"]["DataType"]) + " yet") if state["ProblemType"]["Sparse"] != 0: - reject(state, "MultipleBufferSingleKernel not support sparse yet") + reject(state, printRejectionReason, "MultipleBufferSingleKernel not support sparse yet") #Need to force disabling PreloadKernArgs if compiler does not support #Can not just reject the solution since the user library may find any solutions if state["PreloadKernArgs"]: - hipccver = globalParameters['HipClangVersion'].split(".") - hipccMaj = int(hipccver[0]) - hipccPatch = int(hipccver[2].split("-")[0]) - if not (hipccMaj >= 6 and hipccPatch >= 32650 and (isa == (9, 0, 10) or isa[:2] == (9, 4))): + if not (rocmVersion.major >= 6 and rocmVersion.patch >= 32650 and (isa == (9, 0, 10) or isa[:2] == (9, 4))): #print("Force to Disable PreloadKernArgs since this hipcc version doesn't support",) state["PreloadKernArgs"] = 0 - ######################################## - # create a dictionary with booleans on whether to include parameter in name - @staticmethod - def getMinNaming(objs): - nonCKObjs = [obj for obj in objs if not isCustomKernelConfig(obj)] - - # early return - if len(nonCKObjs) == 0: - return {} - - # determine keys - requiredParameters = {} - if isinstance(nonCKObjs[0], Solution): - keys = list(nonCKObjs[0]._state.keys()) - else: - keys = list(nonCKObjs[0].keys()) - # only 1, rather than name being nothing, it'll be everything - if len(nonCKObjs) == 1: - for key in keys: - if key in list(validParameters.keys()): - requiredParameters[key] = False - else: - for key in keys: - required = False - if key in list(validParameters.keys()): - for i in range(1, len(nonCKObjs)): - if nonCKObjs[0][key] != nonCKObjs[i][key]: - required = True - break - if required: - requiredParameters[key] = True - else: - requiredParameters[key] = False - - requiredParameters["GlobalSplitU"] = True - requiredParameters["WorkGroupMapping"] = True - - if "MatrixInstM" in nonCKObjs[0]._state: - # Use MIWaveGroup and MIWaveTile instead of WG and MT - requiredParameters["MIWaveTile"] = True - requiredParameters["ThreadTile"] = False - - requiredParameters["ProblemType"] = False # always prepended - requiredParameters["MacroTile0"] = False # always prepended - requiredParameters["MacroTile1"] = False # always prepended - requiredParameters["DepthU"] = False # always prepended - requiredParameters["MatrixInstruction"] = False # always prepended - requiredParameters["MatrixInstM"] = False # always prepended - requiredParameters["MatrixInstN"] = False # always prepended - requiredParameters["MatrixInstK"] = False # always prepended - requiredParameters["MatrixInstB"] = False # always prepended - requiredParameters["MatrixInstBM"] = False # always prepended - requiredParameters["MatrixInstBN"] = False # always prepended - requiredParameters["CustomKernelName"] = False # Will not affect naming - - requiredParameters["Kernel"] = True # distinguish kernels from solutions - # for single-source compilation - return requiredParameters - - ######################################## - @ staticmethod - def getKeyNoInternalArgs(state): - state_copy = deepcopy(state) - - state_copy["ProblemType"]["GroupedGemm"] = False - - if globalParameters["SplitGSU"]: - state_copy["GlobalSplitU"] = "M" if (state_copy["GlobalSplitU"] > 1) else state_copy["GlobalSplitU"] - elif state["GlobalSplitU"] > 0: - state_copy["GlobalSplitU"] = "M" - state_copy["WorkGroupMapping"] = "M" - state_copy["WorkGroupMappingXCC"] = "M" - state_copy["WorkGroupMappingXCCGroup"] = "M" - state_copy["StaggerU"] = "M" - state_copy["StaggerUStride"] = "M" - state_copy["StaggerUMapping"] = "M" - state_copy["GlobalSplitUCoalesced"] = "M" - state_copy["GlobalSplitUWorkGroupMappingRoundRobin"] = "M" - - return state_copy - - @ staticmethod - def getNameFull(state): - requiredParameters = {} - for key in state: - if key in list(validParameters.keys()): - requiredParameters[key] = True - if "MatrixInstM" in state: - # Use MIWaveGroup and MIWaveTile instead of WG and MT - requiredParameters["MIWaveTile"] = True - requiredParameters["ThreadTile"] = False - return Solution.getNameMin(state, requiredParameters) - - ######################################## - # Get Name Min - @ staticmethod - def getNameMin(state, requiredParameters, ignoreInternalArgs = False): - if isCustomKernelConfig(state): - return state["CustomKernelName"] - - components = [] - - backup = state["ProblemType"]["GroupedGemm"] - if ignoreInternalArgs: - state["ProblemType"]["GroupedGemm"] = False - - if "ProblemType" in state: - components.append(f'{str(state["ProblemType"])}') - # name += str(state["ProblemType"]) + "_" - - if ignoreInternalArgs: - state["ProblemType"]["GroupedGemm"] = backup - - if "MacroTile0" in state \ - and "MacroTile1" in state \ - and "DepthU" in state: - components.append(f'{Solution.getParameterNameAbbreviation("MacroTile")}{state["MacroTile0"]}x{state["MacroTile1"]}x{state["DepthU"]}') - - if "MatrixInstM" in state: - components.append(f'{Solution.getParameterNameAbbreviation("MatrixInstruction")}{state["MatrixInstM"]}x{state["MatrixInstN"]}x{state["MatrixInstB"]}') - - backup = state["GlobalSplitU"] - - if ignoreInternalArgs: - if globalParameters["SplitGSU"]: - state["GlobalSplitU"] = "M" if (state["GlobalSplitU"] > 1) else state["GlobalSplitU"] - elif state["GlobalSplitU"] > 0: - requiredParameters["GlobalSplitU"] = False - requiredParameters["WorkGroupMapping"] = False - requiredParameters["WorkGroupMappingXCC"] = False - requiredParameters["WorkGroupMappingXCCGroup"] = False - requiredParameters["StaggerU"] = False - requiredParameters["StaggerUStride"] = False - requiredParameters["StaggerUMapping"] = False - requiredParameters["GlobalSplitUCoalesced"] = False - requiredParameters["GlobalSplitUWorkGroupMappingRoundRobin"] = False - - useWaveTile, useThreadTile = requiredParameters.get("MIWaveTile", False), requiredParameters.get("ThreadTile", False) - - if 'MatrixInstM' in state: - requiredParameters["MIWaveTile"] = True - requiredParameters["ThreadTile"] = False - else: - requiredParameters["MIWaveTile"] = False - requiredParameters["ThreadTile"] = True - - components.append('SN') - for key in sorted(state.keys()): - if key in requiredParameters and key[0] != '_': - if requiredParameters[key] and key != "CustomKernelName": - components.append(f'{Solution.getParameterNameAbbreviation(key)}{Solution.getParameterValueAbbreviation(key, state[key])}') - - state["GlobalSplitU"] = backup - requiredParameters["GlobalSplitU"] = True - requiredParameters["WorkGroupMapping"] = True - requiredParameters["WorkGroupMappingXCC"] = True - requiredParameters["WorkGroupMappingXCCGroup"] = True - requiredParameters["StaggerU"] = True - requiredParameters["StaggerUStride"] = True - requiredParameters["StaggerUMapping"] = True - requiredParameters["GlobalSplitUCoalesced"] = True - requiredParameters["GlobalSplitUWorkGroupMappingRoundRobin"] = True - requiredParameters["MIWaveTile"] = useWaveTile - requiredParameters["ThreadTile"] = useThreadTile - - return '_'.join(components) - - ######################################## - # create a dictionary of lists of parameter values - @staticmethod - def getSerialNaming(objs): - data = {} - for obj in objs: - for paramName in sorted(obj.keys()): - if paramName in validParameters.keys(): - paramValue = obj[paramName] - if paramName in data: - if paramValue not in data[paramName]: - data[paramName].append(paramValue) - else: - data[paramName] = [ paramValue ] - maxObjs = 1 - for paramName in data: - if not isinstance(data[paramName][0], dict): - data[paramName] = sorted(data[paramName]) - maxObjs *= len(data[paramName]) - numDigits = len(str(maxObjs)) - return [ data, numDigits ] - - ######################################## - # Get Name Serial - @ staticmethod - def getNameSerial(state, serialNaming): - data = serialNaming[0] - numDigits = serialNaming[1] - - serial = 0 - multiplier = 1 - for paramName in sorted(state.keys()): - if paramName in list(validParameters.keys()): - paramValue = state[paramName] - paramData = data[paramName] - paramNameMultiplier = len(paramData) - if paramValue in paramData: - paramValueIdx = paramData.index(paramValue) - serial += paramValueIdx * multiplier - multiplier *= paramNameMultiplier - name = "%s%0*u" % ("S" if isinstance(state, Solution) else "K", \ - numDigits, serial) - return name - ######################################## @ staticmethod @@ -4463,60 +3151,10 @@ def getParametersIndented(state, indent): s += "%s%s: %s\n" % (indent, str(key), str(state[key])) return s - ######################################## - @ staticmethod - @ lru_cache(maxsize=None) - def getParameterNameAbbreviation( name: str ): - return ''.join(c for c in name if c.isupper()) - - ######################################## class NonprimitiveParameterValueException(Exception): pass - @ staticmethod - @ lru_cache(maxsize=None) - def getPrimitiveParameterValueAbbreviation(key, value): - if isinstance(value, str): - return Solution.getParameterNameAbbreviation(value) - elif isinstance(value, bool): - return "1" if value else "0" - elif isinstance(value, int): - if value >= 0: - return "%u" % value - else: # -1 -> n1 - return "n%01u" % abs(value) - elif isinstance(value, ProblemType): - return str(value) - elif isinstance(value, float): - val1 = int(value) - val2 = int(round(value*100)) - int(value)*100 - if val2 > 0: - s = "%dp%s" % (val1,str(val2).zfill(2)) - else: - s = "%d" % (val1) - return s - - ######################################## - - @ staticmethod - def getParameterValueAbbreviation( key, value ): - if key == "ISA": - return f"{value[0]}{value[1]}{value[2]:x}" - - compositieTypes = (dict, list, tuple,) - - if not isinstance(value, compositieTypes): - return Solution.getPrimitiveParameterValueAbbreviation(key, value) - elif isinstance(value, tuple): - return ''.join(str(v) for v in value) - elif isinstance(value, list): - return '_'.join(Solution.getParameterValueAbbreviation(key, v) for v in value) - elif isinstance(value, dict): - return "_".join(f"{pos:d}{k:d}" for pos,k in value.items()) - else: - printExit('Parameter {key}={value} is new object type ({t})'.format(key=key, value=value, t=type(value))) - return str(value) ########################## # make class look like dict @@ -4538,7 +3176,7 @@ def __setitem__(self, key, value): def __str__(self): if self._name is None: - self._name = Solution.getNameFull(self._state) + self._name = getNameFull(self._state, self.splitGSU) return self._name def __repr__(self): diff --git a/tensilelite/Tensile/SolutionStructs/Utilities.py b/tensilelite/Tensile/SolutionStructs/Utilities.py new file mode 100644 index 0000000000..659a2a83a2 --- /dev/null +++ b/tensilelite/Tensile/SolutionStructs/Utilities.py @@ -0,0 +1,165 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +import sys +import pprint +from typing import Dict, Optional + +from Tensile.Common import IsaVersion, IsaInfo, print1, print2 +from Tensile.Common.ValidParameters import validMFMA +from Tensile.TensileInstructions import DataType + +def reject(state: dict, printSolutionRejectionReason: bool = True, *args) -> bool: + """ + Reject a solution based on its internal state. + + Args: + state: The state of the solution. + printSolutionRejectionReason: If True, print the rejection reason. + *args: Additional arguments to print if rejection occurs. + + Returns: + True if the solution is rejected, False otherwise. + """ + if state and "NoReject" in state and state["NoReject"]: + return False + if printSolutionRejectionReason: + sys.stdout.write("\nreject: ") + for a in args: + print(a) + #traceback.print_stack(None, 2) + solutionIndex = state["SolutionIndex"] if (state != None and "SolutionIndex" in state) else -1 + if solutionIndex != -1: + # If we have valid solutionIndex, this means we are during TensileCreateLibrary stage + # In this stage, all solutions in the logic should be valid + # So if any rejection happens, print the warning for further check + # This will be done only when --global-parameters=PrintSolutionRejectionReason=True + solutionNameMin = state["SolutionNameMin"] if ("SolutionNameMin" in state) else None + # if we don't have SolutionNameMin, we simply use the problemTypeName + solutionNameMin = str(state["ProblemType"]) if (solutionNameMin == None) else solutionNameMin + raise Exception("!! Warning: Any rejection of a LibraryLogic is not expected, please check. \ + SolutionIndex: %d (or SolutionName/ProblemType: %s)"%(solutionIndex, solutionNameMin)) + if state != None: + state["Valid"] = False + return True + +def matrixInstructionToMIParameters( + mi: list, + isa: IsaVersion, + wavefrontSize: int, + problemType: dict, + workGroup: Optional[list], + isaInfoMap: Dict[IsaVersion, IsaInfo] + ): + """ + Converts a 9-item matrix instruction into the associated 4-item representation and + populates supporting MI parameters. + + Args: + mi: The matrix instruction to convert. Must have length 9. + isa: The ISA tuple. + wavefrontSize: The wavefront size. Typically "WavefrontSize" in a solution. + problemType: The problem type dictionary. Typically "ProblemType" in a solution. + """ + print1(f">> --DBG-- Converting MatrixInstruction {mi} to MI parameters") + + if len(mi) != 9: + raise ValueError(f"MatrixInstruction must be 9 items long to convert into MI" + f" Parameters, found {mi} with length {len(mi)}") + + result = {} + result["ISA"] = isa + + # Enable F32 XDL math operation only when the input type is f32. + enableF32xdl = ( + "F32XdlMathOp" in problemType + and not problemType["F32XdlMathOp"].isSingle() + and problemType["DataType"].isSingle() + ) + result["EnableF32XdlMathOp"] = enableF32xdl + + mi4 = [mi[0], mi[1], mi[2], mi[3]] + result["MatrixInstruction"] = mi4 + result["EnableMatrixInstruction"] = True + result["MatrixInstM"] = mi[0] + result["MatrixInstN"] = mi[1] + result["MatrixInstK"] = mi[2] + result["MatrixInstB"] = mi[3] + + waves = mi[7]* mi[8] + wg0 = mi[4] * mi[0] * mi[7] + + result["WavefrontSize"] = wavefrontSize + if workGroup: + # Some Solutions used during benchmarking don't have WorkGroup set. + result["WorkGroup"] = [wg0, waves*wavefrontSize // wg0, workGroup[2]] + result["ThreadTile"] = [1, 1] # dummy + + isSparse = problemType.get("Sparse", 0) + miDataType = DataType( + problemType["DataType"] + if not enableF32xdl + else problemType["F32XdlMathOp"] + ) + + result["MFMA_BF16_1K"] = ( + not isSparse + and isaInfoMap[isa].asmCaps["HasMFMA"] + and not (miDataType.toChar() in validMFMA and mi4 in validMFMA[miDataType.toChar()]) + and miDataType.isBFloat16() + and mi4 in validMFMA["B1k"] + ) + + # set MIBlock + MIBlockBM = wg0 // mi[0] + MIBlockBM = min(MIBlockBM, mi[3]) + MIBlockBN = mi[3] // MIBlockBM + result["MatrixInstBM"] = MIBlockBM + result["MatrixInstBN"] = MIBlockBN + result["MIBlock"] = [mi[0], mi[1], mi[2], mi[3], MIBlockBM, MIBlockBN] + + # set MIWaveGroup + miwg0 = min((wg0 // mi[0]) // MIBlockBM, waves) + result['MIWaveGroup'] = [miwg0, waves // miwg0] + + # set MIWaveTile + result['MIWaveTile'] = [mi[5], mi[6]] + + # set MIInputPerThread + hasMFMA = isaInfoMap[isa].asmCaps["HasMFMA"] + hasWMMA = isaInfoMap[isa].asmCaps["HasWMMA"] + + result['MIInputPerThread'] = mi[0] * mi[2] * mi[3] // wavefrontSize + if (not hasMFMA) and hasWMMA and (isa[0] == 10 or isa[0] == 11): + result['MIInputPerThread'] = mi[2] + + sparseA = False if not isSparse else False if isSparse == 2 else True + sparseB = False if not isSparse else True if isSparse == 2 else False + result['MIInputPerThreadA'] = result['MIInputPerThread'] if not sparseA else result['MIInputPerThread'] // 2 + result['MIInputPerThreadB'] = result['MIInputPerThread'] if not sparseB else result['MIInputPerThread'] // 2 + result['MIInputPerThreadMetadata'] = result['MIInputPerThread'] if not isSparse else result['MIInputPerThread'] // 8 + result['Sparse'] = isSparse + + print2(f">> MI Parameters: {pprint.pformat(result)}") + return result diff --git a/tensilelite/Tensile/SolutionStructs/__init__.py b/tensilelite/Tensile/SolutionStructs/__init__.py new file mode 100644 index 0000000000..cd9b96908c --- /dev/null +++ b/tensilelite/Tensile/SolutionStructs/__init__.py @@ -0,0 +1,4 @@ +from .Naming import * +from .Solution import * +from .Problem import * +from .Utilities import * diff --git a/tensilelite/Tensile/Source/ReductionTemplate.h b/tensilelite/Tensile/Source/ReductionTemplate.h index ee61de51e9..f77729b550 100644 --- a/tensilelite/Tensile/Source/ReductionTemplate.h +++ b/tensilelite/Tensile/Source/ReductionTemplate.h @@ -2,7 +2,7 @@ * * MIT License * - * Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/tensilelite/Tensile/Tensile.py b/tensilelite/Tensile/Tensile.py index 81afc23b3b..8ecae196cd 100644 --- a/tensilelite/Tensile/Tensile.py +++ b/tensilelite/Tensile/Tensile.py @@ -26,24 +26,27 @@ print("This file can no longer be run as a script. Run 'Tensile/bin/Tensile' instead.") exit(1) -import joblib import os +import subprocess import sys import argparse -from .Common import globalParameters, print1, printExit, printWarning, ensurePath, \ - assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR -from .Toolchain.Assembly import AssemblyToolchain -from .Toolchain.Source import SourceToolchain -from .Toolchain.Validators import validateToolchain, ToolchainDefaults -from .Utilities.Decorators.Profile import profile -from . import BenchmarkProblems -from . import ClientWriter -from . import LibraryIO -from . import LibraryLogic + from datetime import datetime from pathlib import Path - -import subprocess +from typing import Dict + +from Tensile.Common import globalParameters, print1, printExit, printWarning, ensurePath, \ + assignGlobalParameters, restoreDefaultGlobalParameters, HR, __version__, LIBRARY_LOGIC_DIR, \ + detectGlobalCurrentISA, verbosity, IsaInfo, makeIsaInfoMap, isaToGfx, makeDebugConfig, \ + makeDepthUConfig, DebugConfig, DepthUConfig +from Tensile.Toolchain.Assembly import AssemblyToolchain, makeAssemblyToolchain +from Tensile.Toolchain.Source import SourceToolchain, makeSourceToolchain +from Tensile.Toolchain.Validators import validateToolchain, ToolchainDefaults +from Tensile.Utilities.Decorators.Profile import profile +from Tensile import BenchmarkProblems +from Tensile import ClientWriter +from Tensile import LibraryIO +from Tensile import LibraryLogic ############################################################################### # Execute Steps in Config @@ -59,7 +62,11 @@ def executeStepsInConfig( outputPath: Path, asmToolchain: AssemblyToolchain, srcToolchain: SourceToolchain, - cCompiler: str + isaInfoMap: Dict[str, IsaInfo], + cCompiler: str, + debugConfig: DebugConfig, + depthUConfig: DepthUConfig, + deviceId: int ): """Conducts the steps in the provided ``config`` according to the Tensile workflow. @@ -84,7 +91,20 @@ def executeStepsInConfig( # Benchmark Problems ############################################################################## if "BenchmarkProblems" in config: - BenchmarkProblems.main(config["BenchmarkProblems"], config["UseCache"], asmToolchain, srcToolchain, cCompiler, outputPath, buildTmpPath) + BenchmarkProblems.main( + config["BenchmarkProblems"], + config["UseCache"], + asmToolchain, + srcToolchain, + cCompiler, + outputPath, + buildTmpPath, + config["ShortNames"], + debugConfig, + depthUConfig, + deviceId, + isaInfoMap, + ) print1("") ############################################################################## @@ -101,7 +121,16 @@ def executeStepsInConfig( libraryLogicConfig = config["LibraryLogic"] else: libraryLogicConfig = {} - LibraryLogic.main(libraryLogicConfig, srcToolchain.compiler, outputPath) + LibraryLogic.main( + libraryLogicConfig, + srcToolchain.compiler, + outputPath, + debugConfig.splitGSU, + debugConfig.printSolutionRejectionReason, + debugConfig.printIndexAssignmentInfo, + depthUConfig, + isaInfoMap, + ) print1("") else: print1("# LibraryLogic already done.") @@ -115,7 +144,15 @@ def executeStepsInConfig( libraryClientConfig = config["LibraryClient"] else: libraryClientConfig = {} - ClientWriter.main(libraryClientConfig, srcToolchain.compiler, cCompiler, outputPath) + ClientWriter.main( + libraryClientConfig, + asmToolchain.assembler, + cCompiler, + isaInfoMap, + outputPath, + deviceId, + config["ShortNames"] + ) print1("") @@ -134,7 +171,7 @@ def splitExtraParameters(par): value = eval(value) return (key, value) - argParser.add_argument("-d", "--device", dest="device", type=int, \ + argParser.add_argument("-d", "--device", dest="device", default=0, type=int, \ help="override which device to benchmark") argParser.add_argument("-p", "--platform", dest="platform", type=int, \ help="override which OpenCL platform to benchmark") @@ -172,9 +209,6 @@ def argUpdatedGlobalParameters(args): """ rv = {} # override config with command-line options - if args.device: - print1("# Command-line override: Device") - rv["Device"] = args.device if args.platform: print1("# Command-line override: Platform") rv["Platform"] = args.platform @@ -184,15 +218,9 @@ def argUpdatedGlobalParameters(args): if args.CodeObjectVersion: print1("# Command-line override: CodeObjectVersion") rv["CodeObjectVersion"] = args.CodeObjectVersion - if args.verbose: - print1("# Command-line override: PrintLevel") - rv["PrintLevel"] = 2 if args.debug: print1("# Command-line override: Debug") - rv["PrintLevel"] = 2 rv["CMakeBuildType"] = "Debug" - if args.shortNames: - rv["ShortNames"] = True if args.client_lock: rv["ClientExecutionLockPath"] = args.client_lock if args.prebuilt_client: @@ -332,13 +360,15 @@ def Tensile(userArgs): addCommonArguments(argParser) args = argParser.parse_args(userArgs) - configPaths = args.ConfigFile altFormat = args.AlternateFormat useCache = args.useCache outputPath = Path(ensurePath(os.path.abspath(args.OutputPath))) print1(f"# OutputPath: {str(outputPath)}") + global verbosity + verbosity = 2 if (args.debug or args.verbose) else 1 + if altFormat and len(configPaths) > 2: printExit("Only 1 or 2 config_files are accepted for the alternate config format: " "the alternate config file and an optional size list") @@ -393,7 +423,7 @@ def Tensile(userArgs): config["UseCache"] = useCache globalParameters["ConfigPath"] = configPaths - device_id = config["GlobalParameters"].get("Device", globalParameters["Device"]) + device_id = config["GlobalParameters"].get("Device", int(args.device)) UseEffLike = config["GlobalParameters"].get("UseEffLike", globalParameters["UseEffLike"]) if 'LibraryLogic' in config and UseEffLike: @@ -409,14 +439,28 @@ def Tensile(userArgs): print(f"Successfully retrieve Max frequency: {max_frequency} for device {device_id}") store_max_frequency(max_frequency) - cxxCompiler, cCompiler, assembler, offloadBundler = validateToolchain(args.CxxCompiler, args.CCompiler, args.Assembler, args.OffloadBundler) - assignGlobalParameters(config.get("GlobalParameters", {}), cxxCompiler) + cxxCompiler, cCompiler, _, offloadBundler = validateToolchain(args.CxxCompiler, args.CCompiler, args.Assembler, args.OffloadBundler) + currentIsa = detectGlobalCurrentISA(device_id) + isaInfoMap = makeIsaInfoMap([currentIsa], cxxCompiler) + assignGlobalParameters(config.get("GlobalParameters", {}), isaInfoMap) + + asmToolchain = makeAssemblyToolchain( + cxxCompiler, + offloadBundler, + args.CodeObjectVersion, + ) + srcToolchain = makeSourceToolchain( + cxxCompiler, + offloadBundler, + ) + overrideParameters = argUpdatedGlobalParameters(args) - asmToolchain= AssemblyToolchain(assembler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["CodeObjectVersion"]) - srcToolchain= SourceToolchain(cxxCompiler, offloadBundler, globalParameters["BuildIdKind"], globalParameters["AsanBuild"], globalParameters["SaveTemps"]) + if "ShortNames" not in config: + config["ShortNames"] = args.shortNames - overrideParameters = argUpdatedGlobalParameters(args) + debugConfig = makeDebugConfig(config["GlobalParameters"]) + depthUConfig = makeDepthUConfig(config["GlobalParameters"]) for key, value in overrideParameters.items(): print("Overriding {0}={1}".format(key, value)) @@ -425,7 +469,7 @@ def Tensile(userArgs): if "MaxFileName" in globalParameters or "MaxFileName" in config: printWarning("MaxFileName is no longer configurable, it will be automatically set to 64") - executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, cCompiler) + executeStepsInConfig(config, outputPath, asmToolchain, srcToolchain, isaInfoMap, cCompiler, debugConfig, depthUConfig, device_id) def TensileConfigPath(*args): return os.path.join(os.path.dirname(os.path.realpath(__file__)), "Configs", *args) diff --git a/tensilelite/Tensile/TensileClientConfig.py b/tensilelite/Tensile/TensileClientConfig.py index da6d2f1377..f5eb55bbaa 100644 --- a/tensilelite/Tensile/TensileClientConfig.py +++ b/tensilelite/Tensile/TensileClientConfig.py @@ -25,7 +25,7 @@ from . import ClientWriter from . import LibraryIO from .Contractions import ProblemType as ContractionsProblemType -from .SolutionStructs import ProblemSizes, ProblemType +from Tensile.SolutionStructs.Problem import ProblemType, ProblemSizes from .Common import globalParameters, print1, printExit, printWarning, assignGlobalParameters, \ restoreDefaultGlobalParameters, HR, __version__ from .Tensile import addCommonArguments, argUpdatedGlobalParameters diff --git a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py index e7fd5b6ced..c35b96920b 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py +++ b/tensilelite/Tensile/TensileCreateLibrary/ParseArguments.py @@ -201,7 +201,6 @@ def parseArguments(input: Optional[List[str]] = None) -> Dict[str, Any]: if args.CmakeCxxCompiler: os.environ["CMAKE_CXX_COMPILER"] = args.CmakeCxxCompiler arguments["ShortNames"] = args.ShortNames - arguments["CodeFromFiles"] = False arguments["LogicFormat"] = args.LogicFormat arguments["LibraryFormat"] = args.LibraryFormat if args.no_enumerate: diff --git a/tensilelite/Tensile/TensileCreateLibrary/Run.py b/tensilelite/Tensile/TensileCreateLibrary/Run.py index 54f93810f2..72e3ba87e0 100644 --- a/tensilelite/Tensile/TensileCreateLibrary/Run.py +++ b/tensilelite/Tensile/TensileCreateLibrary/Run.py @@ -29,26 +29,34 @@ import shutil from pathlib import Path from timeit import default_timer as timer -from typing import List, NamedTuple, Optional, Sequence, Union +from typing import List, NamedTuple, Optional, Union from Tensile import SOURCE_PATH, LibraryIO from Tensile.Common import ( - HR, - CHeader, - IsaVersion, - ParallelMap2, - SemanticVersion, - architectureMap, assignGlobalParameters, + CHeader, + DebugConfig, + DepthUConfig, ensurePath, globalParameters, + gfxToIsa, + HR, isaToGfx, + IsaVersion, + makeIsaInfoMap, + ParallelMap2, print1, print2, + printWarning, printExit, + printWarning, state, + SUPPORTED_GFX, tqdm, + verbosity, ) +from Tensile.SolutionStructs.Naming import getKernelFileBase, getKeyNoInternalArgs, getMinNaming, getSerialNaming + from Tensile.CustomYamlLoader import load_logic_gfx_arch from Tensile.KernelWriterAssembly import KernelWriterAssembly from Tensile.KernelWriterBase import ( @@ -58,13 +66,13 @@ from Tensile.SolutionLibrary import MasterSolutionLibrary from Tensile.SolutionStructs import Solution from Tensile.TensileInstructions import TensileInstructions -from Tensile.Toolchain.Assembly import AssemblyToolchain, buildAssemblyCodeObjectFiles -from Tensile.Toolchain.Source import SourceToolchain, buildSourceCodeObjectFiles +from Tensile.Toolchain.Assembly import makeAssemblyToolchain, buildAssemblyCodeObjectFiles +from Tensile.Toolchain.Source import makeSourceToolchain, SourceToolchain, buildSourceCodeObjectFiles from Tensile.Toolchain.Validators import ( ToolchainDefaults, - getVersion, validateToolchain, ) +from Tensile.Toolchain.Component import Assembler from Tensile.Utilities.Decorators.Profile import profile from Tensile.Utilities.Decorators.Timing import timing @@ -81,15 +89,15 @@ class KernelCodeGenResult(NamedTuple): wavefrontSize: int -def processKernelSource(kernelWriterAssembly, ti, kernel) -> KernelCodeGenResult: +def processKernelSource(kernelWriterAssembly, ti, useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, kernel) -> KernelCodeGenResult: """ Generate source for a single kernel. Returns (error, source, header, kernelName). """ kernelWriter = kernelWriterAssembly kernelWriter.setTensileInstructions(ti) - asmFilename = kernelWriter.getKernelFileBase(kernel) - err, src = kernelWriter.getSourceFileString(kernel) + asmFilename = getKernelFileBase(useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, kernel) + err, src = kernelWriter.getSourceFileString(kernel, useShortNames) header = kernelWriter.getHeaderFileString(kernel) objFilename = kernel._state.get("codeObjectFile", None) @@ -98,14 +106,14 @@ def processKernelSource(kernelWriterAssembly, ti, kernel) -> KernelCodeGenResult ) -def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, globalParameters): +def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, printLevel: bool, splitGSU: bool): removeKernels = [] removeKernelNames = [] removeSolutions = [] removeResults = [] for kernIdx, r in ( - tqdm(enumerate(results)) if globalParameters["PrintLevel"] > 1 else enumerate(results) + tqdm(enumerate(results)) if printLevel > 1 else enumerate(results) ): if r.err != 0: if not errorTolerant: @@ -116,7 +124,7 @@ def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, ) print(kernels[kernIdx]["SolutionNameMin"]) removeKernels.append(kernels[kernIdx]) - kName = Solution.getKeyNoInternalArgs(kernels[kernIdx]) + kName = getKeyNoInternalArgs(kernels[kernIdx], splitGSU) if kName not in removeKernelNames: removeKernelNames.append(kName) removeResults.append(results[kernIdx]) @@ -129,12 +137,12 @@ def removeInvalidSolutionsAndKernels(results, kernels, solutions, errorTolerant, for solution in ( tqdm(solutions, "Finding invalid solutions") - if globalParameters["PrintLevel"] > 1 + if printLevel > 1 else solutions ): solutionKernels = solution.getKernels() for kernel in solutionKernels: - kName = Solution.getKeyNoInternalArgs(kernel) + kName = getKeyNoInternalArgs(kernel, splitGSU) if kName in removeKernelNames: removeSolutions.append(solution) break @@ -174,9 +182,8 @@ def writeHelpers( kernelHeaderFile.write(CHeader) kernelSourceFile.write('#include "Kernels.h"\n') kernelHeaderFile.write("#pragma once\n") - if globalParameters["RuntimeLanguage"] == "HIP": - kernelHeaderFile.write("#include \n") - kernelHeaderFile.write("#include \n\n") + kernelHeaderFile.write("#include \n") + kernelHeaderFile.write("#include \n\n") kernelHeaderFile.write('#include "KernelHeader.h"\n\n') HeaderText = "" for ko in kernelHelperObjs: @@ -197,10 +204,14 @@ def writeSolutionsAndKernels( kernels, kernelHelperObjs, kernelWriterAssembly, + splitGSU: bool, + cmdlineArchs: List[str], + kernelSerialNaming, + kernelMinNaming, errorTolerant=False, generateSourcesAndExit=False, compress=True, - fromTensile=False, + useShortNames=False, ): codeObjectFiles = [] @@ -221,8 +232,10 @@ def writeSolutionsAndKernels( visited = set() duplicates = 0 for k in asmKernels: - base = kernelWriterAssembly.getKernelFileBase(k) + base = getKernelFileBase(useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, k) k.duplicate = True if base in visited else False + if not k.duplicate: + k["BaseName"] = base duplicates += k.duplicate print2(f"Duplicate: {base}") visited.add(base) @@ -232,16 +245,22 @@ def writeSolutionsAndKernels( numKernels = len(asmKernels) assert numKernels == numAsmKernels, "Only assembly kernels are supported in TensileLite" asmIter = zip( - itertools.repeat(kernelWriterAssembly), itertools.repeat(TensileInstructions()), asmKernels + itertools.repeat(kernelWriterAssembly), + itertools.repeat(TensileInstructions()), + itertools.repeat(useShortNames), + itertools.repeat(splitGSU), + itertools.repeat(kernelMinNaming), + itertools.repeat(kernelSerialNaming), + asmKernels ) - asmResults = ParallelMap2(processKernelSource, asmIter, "Generating assembly kernels") + asmResults = ParallelMap2(processKernelSource, asmIter, "Generating assembly kernels", return_as="list") removeInvalidSolutionsAndKernels( - asmResults, asmKernels, solutions, errorTolerant, globalParameters + asmResults, asmKernels, solutions, errorTolerant, verbosity, splitGSU ) def assemble(ret): p, isa, wavefrontsize = ret - asmToolchain.assemble(str(p), str(p.with_suffix(".o")), isaToGfx(isa), wavefrontsize) + asmToolchain.assembler(isaToGfx(isa), wavefrontsize, str(p), str(p.with_suffix(".o"))) unaryWriteAssembly = functools.partial(writeAssembly, assemblyTmpPath) compose = lambda *F: functools.reduce(lambda f, g: lambda x: f(g(x)), F) @@ -258,10 +277,22 @@ def assemble(ret): if not generateSourcesAndExit: codeObjectFiles += buildAssemblyCodeObjectFiles( - asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress + asmToolchain.linker, + asmToolchain.bundler, + globalParameters["ROCmLdPath"], + asmKernels, + destLibPath, + assemblyTmpPath, + compress, ) buildSourceCodeObjectFiles( - srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, fromTensile + srcToolchain.compiler, + srcToolchain.bundler, + destLibPath, + objectTmpPath, + outputPath, + srcKernelFile, + cmdlineArchs, ) return codeObjectFiles, numKernels @@ -274,10 +305,12 @@ def writeSolutionsAndKernelsTCL( kernels, kernelHelperObjs, kernelWriterAssembly, + cmdlineArchs: List[str], + kernelSerialNaming, + kernelMinNaming, compress=True, - fromTensile=False, + useShortNames=False, ): - outputPath = Path(outputPath) destLibPath = ensurePath( outputPath / "library" @@ -294,8 +327,10 @@ def writeSolutionsAndKernelsTCL( visited = set() duplicates = 0 + splitGSU = False for k in asmKernels: - base = kernelWriterAssembly.getKernelFileBase(k) + base = getKernelFileBase(useShortNames, splitGSU, kernelMinNaming, kernelSerialNaming, k) + k["BaseName"] = base k.duplicate = True if base in visited else False duplicates += k.duplicate print2(f"Duplicate: {base}") @@ -306,11 +341,18 @@ def writeSolutionsAndKernelsTCL( def assemble(ret): p, isa, wavefrontsize = ret - asmToolchain.assemble(str(p), str(p.with_suffix(".o")), isaToGfx(isa), wavefrontsize) + asmToolchain.assembler(isaToGfx(isa), wavefrontsize, str(p), str(p.with_suffix(".o"))) unaryProcessKernelSource = functools.partial( - processKernelSource, kernelWriterAssembly, TensileInstructions() + processKernelSource, + kernelWriterAssembly, + TensileInstructions(), + useShortNames, + splitGSU, + kernelMinNaming, + kernelSerialNaming ) + unaryWriteAssembly = functools.partial(writeAssembly, assemblyTmpPath) compose = lambda *F: functools.reduce(lambda f, g: lambda x: f(g(x)), F) ret = ParallelMap2( @@ -318,34 +360,33 @@ def assemble(ret): uniqueAsmKernels, "Generating assembly kernels", multiArg=False, + return_as="list" ) buildAssemblyCodeObjectFiles( - asmToolchain, asmKernels, kernelWriterAssembly, destLibPath, assemblyTmpPath, compress + asmToolchain.linker, + asmToolchain.bundler, + globalParameters["ROCmLdPath"], + asmKernels, + destLibPath, + assemblyTmpPath, + compress, ) writeHelpers(outputPath, kernelHelperObjs, KERNEL_HELPER_FILENAME_CPP, KERNEL_HELPER_FILENAME_H) srcKernelFile = Path(outputPath) / "Kernels.cpp" buildSourceCodeObjectFiles( - srcToolchain, destLibPath, objectTmpPath, outputPath, srcKernelFile, fromTensile + srcToolchain.compiler, + srcToolchain.bundler, + destLibPath, + objectTmpPath, + outputPath, + srcKernelFile, + cmdlineArchs, ) return len(uniqueAsmKernels) -@timing -def getSolutionAndKernelWriters( - solutions, kernels, assembler: str, assemblerVersion: SemanticVersion -): - kernelSerialNaming = Solution.getSerialNaming(kernels) - solutionMinNaming = Solution.getMinNaming(solutions) - kernelMinNaming = Solution.getMinNaming(kernels) - kernelWriterAssembly = KernelWriterAssembly( - kernelMinNaming, kernelSerialNaming, assembler, assemblerVersion - ) - - return (kernelWriterAssembly, kernelMinNaming, solutionMinNaming) - - @timing def copyStaticFiles(outputPath): libraryStaticFiles = [ @@ -370,11 +411,11 @@ def generateKernelObjectsFromSolutions(solutions): kernelHelperObjs = [] kernelNames = set() kernelHelperNames = set() - + splitGSU = False for solution in solutions: solutionKernels = solution.getKernels() for kernel in solutionKernels: - kName = Solution.getKeyNoInternalArgs(kernel) + kName = getKeyNoInternalArgs(kernel, splitGSU) if kName not in kernelNames: kernels.append(kernel) kernelNames.add(kName) @@ -394,7 +435,7 @@ def generateKernelObjectsFromSolutions(solutions): @timing -def generateLogicDataAndSolutions(logicFiles, args, cxxCompiler): +def generateLogicDataAndSolutions(logicFiles, args, assembler: Assembler, isaInfoMap): if ";" in args["Architecture"]: archs = args["Architecture"].split(";") # user arg list format @@ -404,8 +445,20 @@ def generateLogicDataAndSolutions(logicFiles, args, cxxCompiler): solutions = [] masterLibraries = {} nextSolIndex = 0 - - fIter = zip(logicFiles, itertools.repeat(cxxCompiler), itertools.repeat(archs)) + splitGSU = False + printSolutionRejectionReason = False + printIndexAssignmentInfo = False + + fIter = zip( + logicFiles, + itertools.repeat(assembler), + itertools.repeat(splitGSU), + itertools.repeat(printSolutionRejectionReason), + itertools.repeat(printIndexAssignmentInfo), + itertools.repeat(DepthUConfig()), + itertools.repeat(isaInfoMap), + itertools.repeat(args["LazyLibraryLoading"]), + ) def libraryIter(lib: MasterSolutionLibrary): if len(lib.solutions): @@ -493,50 +546,47 @@ def run(): print2("") arguments = parseArguments() + global verbosity + verbosity = arguments["PrintLevel"] outputPath = Path(ensurePath(os.path.abspath(arguments["OutputPath"]))) - cxxCompiler, cCompiler, offloadBundler, assembler, hipconfig = validateToolchain( + cxxCompiler, _, offloadBundler, _, _ = validateToolchain( arguments["CxxCompiler"], arguments["CCompiler"], arguments["OffloadBundler"], arguments["Assembler"], ToolchainDefaults.HIP_CONFIG, ) - print1(f"# HIP Version: {getVersion(hipconfig, regex=r'(.+)')}") - print1(f"# Cxx Compiler: {cxxCompiler} (version {getVersion(cxxCompiler)})") - print1(f"# C Compiler: {cCompiler} (version {getVersion(cCompiler)})") - print1(f"# Assembler: {assembler} (version {getVersion(assembler)})") - print1(f"# Offload Bundler: {offloadBundler} (version {getVersion(offloadBundler)})") - print1(f"# Code Object Version: {arguments['CodeObjectVersion']}") - print1(f"# Architecture(s): {arguments['Architecture']}") - print1(f"# Library Format: {arguments['LibraryFormat']}") - - assignGlobalParameters(arguments, cxxCompiler) - - asmToolchain = AssemblyToolchain( - assembler, offloadBundler, globalParameters["BuildIdKind"], arguments["CodeObjectVersion"] + + if ";" in arguments["Architecture"]: + archs = arguments["Architecture"].split(";") + else: + archs = arguments["Architecture"].split("_") + archs = SUPPORTED_GFX if archs == "all" else archs + + targetIsas = [gfxToIsa(a) for a in archs] + isaInfoMap = makeIsaInfoMap(targetIsas, cxxCompiler) + assignGlobalParameters(arguments, isaInfoMap) + + asmToolchain = makeAssemblyToolchain( + cxxCompiler, + offloadBundler, + arguments["CodeObjectVersion"], + arguments["BuildIdKind"] ) - srcToolchain = SourceToolchain( + srcToolchain = makeSourceToolchain( cxxCompiler, offloadBundler, - globalParameters["BuildIdKind"], - globalParameters["AsanBuild"], - globalParameters["SaveTemps"], + arguments["AsanBuild"], + arguments["BuildIdKind"], + save_temps=False ) + print1(asmToolchain.assembler) + print1(asmToolchain.bundler) + if not os.path.exists(arguments["LogicPath"]): printExit(f"LogicPath {arguments['LogicPath']} doesn't exist") - if ";" in arguments["Architecture"]: - archs = arguments["Architecture"].split(";") - else: - archs = arguments["Architecture"].split("_") - logicArchs = set() - for arch in archs: - if arch in architectureMap: - logicArchs.add(architectureMap[arch]) - else: - printExit("Architecture %s not supported" % arch) - logicExtFormat = ".yaml" if arguments["LogicFormat"] == "yaml": pass @@ -570,13 +620,22 @@ def validLogicFile(p: Path): ] print2(f"# LibraryLogicFiles: {len(logicFiles)}") + for logicFile in logicFiles: print2("# %s" % logicFile) - solutions, masterLibraries = generateLogicDataAndSolutions(logicFiles, arguments, cxxCompiler) + solutions, masterLibraries = generateLogicDataAndSolutions( + logicFiles, arguments, asmToolchain.assembler, isaInfoMap + ) + kernels, kernelHelperObjs, _ = generateKernelObjectsFromSolutions(solutions) - kernelWriterAssembly, kernelMinNaming, _ = getSolutionAndKernelWriters( - solutions, kernels, asmToolchain.assembler, asmToolchain.assemblerVersion + kernelSerialNaming = getSerialNaming(kernels) + kernelMinNaming = getMinNaming(kernels) + kernelWriterAssembly = KernelWriterAssembly( + kernelMinNaming, + kernelSerialNaming, + asmToolchain.assembler, + DebugConfig(), ) copyStaticFiles(outputPath) @@ -588,30 +647,34 @@ def validLogicFile(p: Path): kernels, kernelHelperObjs, kernelWriterAssembly, + archs, + kernelSerialNaming, + kernelMinNaming, + useShortNames=arguments["ShortNames"], compress=arguments["UseCompression"], ) - archs = [ + archs = [ # is this really different than the other archs above? isaToGfx(arch) - for arch in globalParameters["SupportedISA"] - if globalParameters["AsmCaps"][arch]["SupportedISA"] + for arch in targetIsas + if isaInfoMap[arch].asmCaps["SupportedISA"] ] newLibraryDir = ensurePath(os.path.join(outputPath, "library")) - + splitGSU = False for archName, newMasterLibrary in masterLibraries.items(): if archName in archs: - if globalParameters["LazyLibraryLoading"]: + if arguments["LazyLibraryLoading"]: masterFile = os.path.join(newLibraryDir, "TensileLibrary_lazy_" + archName) else: masterFile = os.path.join(newLibraryDir, "TensileLibrary_" + archName) - newMasterLibrary.applyNaming(kernelMinNaming) + newMasterLibrary.applyNaming(splitGSU, kernelMinNaming) LibraryIO.write(masterFile, state(newMasterLibrary), arguments["LibraryFormat"]) for name, lib in newMasterLibrary.lazyLibraries.items(): filename = os.path.join(newLibraryDir, name) - lib.applyNaming(kernelMinNaming) + lib.applyNaming(splitGSU, kernelMinNaming) LibraryIO.write(filename, state(lib), arguments["LibraryFormat"]) - if not globalParameters["KeepBuildTmp"]: + if not arguments["KeepBuildTmp"]: buildTmp = Path(arguments["OutputPath"]).parent / "library" / "build_tmp" if buildTmp.exists() and buildTmp.is_dir(): shutil.rmtree(buildTmp) diff --git a/tensilelite/Tensile/TensileInstructions/Base.py b/tensilelite/Tensile/TensileInstructions/Base.py index 6fa5e42d4c..86397e12ea 100644 --- a/tensilelite/Tensile/TensileInstructions/Base.py +++ b/tensilelite/Tensile/TensileInstructions/Base.py @@ -27,9 +27,11 @@ from dataclasses import dataclass from typing import Tuple -from ..Common import initAsmCaps, initArchCaps, initRegisterCaps, initAsmBugs +from Tensile.Common.Capabilities import initAsmCaps, initArchCaps, initRegisterCaps, initAsmBugs +from Tensile.Common.Types import IsaInfo, IsaVersion from .Formatting import __TI_DEBUG_LEVEL__, printExit +from timeit import default_timer as timer def fastdeepcopy(x): # Note: Some object can't be pickled @@ -48,39 +50,30 @@ def __new__(cls, *args, **kwargs): cls._instance._kernelInfo = {} return cls._instance - def __reduce__(self): - return (TensileInstructions, ()) - - @dataclass - class IsaInfo: - assemblerPath: str - asmCaps: dict - archCaps: dict - regCaps: dict - asmBugs: dict @dataclass class kernelInfo: - isa: Tuple[int, int, int] + isa: IsaVersion wavefrontSize: int = 64 - def init(self, isaVersion: Tuple[int, int, int], assemblerPath: str, debug: bool=False) -> None: + def init(self, isaVersion: IsaVersion, assemblerPath: str, debug: bool=False) -> None: with self._lock: if len(self._kernelInfo) > 1000: self._kernelInfo = _removeIdent(self._kernelInfo) self._kernelInfo[threading.get_ident()] = TensileInstructions.kernelInfo(isa=isaVersion) if isaVersion not in self._isaInfo: # type: ignore + start = timer() asmCaps = initAsmCaps(isaVersion, assemblerPath, debug) archCaps = initArchCaps(isaVersion) regCaps = initRegisterCaps(isaVersion, archCaps) asmBugs = initAsmBugs(asmCaps) - self._isaInfo[isaVersion] = TensileInstructions.IsaInfo(assemblerPath, # type: ignore - asmCaps, archCaps, regCaps, asmBugs) + self._isaInfo[isaVersion] = IsaInfo(asmCaps, archCaps, regCaps, asmBugs) + def setDebugLevel(self, level: int) -> None: __TI_DEBUG_LEVEL__ = level - def setKernelInfo(self, isaVersion: Tuple[int, int, int], wavefrontSize: int) -> None: + def setKernelInfo(self, isaVersion: IsaVersion, wavefrontSize: int) -> None: if isaVersion not in self._isaInfo: # type: ignore import traceback printExit(f"Current isa {str(isaVersion)} not initialized. Initialized isas are {str(self._isaInfo.keys())}, traceback: {traceback.format_stack()}") diff --git a/tensilelite/Tensile/TensileLogic/ParseArguments.py b/tensilelite/Tensile/TensileLogic/ParseArguments.py index 25ce08cc95..70a1e03915 100644 --- a/tensilelite/Tensile/TensileLogic/ParseArguments.py +++ b/tensilelite/Tensile/TensileLogic/ParseArguments.py @@ -23,12 +23,11 @@ ################################################################################ from argparse import ArgumentParser -from typing import Any, Dict from Tensile.Toolchain.Validators import ToolchainDefaults -def parseArguments() -> Dict[str, Any]: +def parseArguments(): """ Returns: A dictionary containing the keys representing options and their values. @@ -40,12 +39,8 @@ def parseArguments() -> Dict[str, Any]: ) argParser.add_argument("LogicPath", help="Path to LibraryLogic.yaml files.") - argParser.add_argument( - "--check-matrix-instruction", - dest="CheckMatrixInstruction", - action="store_true", - help="Checks that matrix instructions are valid for all target ISAs.", - ) + argParser.add_argument("--check", dest="Check", action="store_true", help="Run all checks.") + argParser.add_argument("-v", "--verbose", dest="Verbose", type=int, default=1, choices=[0, 1, 2, 3], help="Set print level with ``--v 2``.") argParser.add_argument( "--jobs", "-j", diff --git a/tensilelite/Tensile/TensileLogic/Run.py b/tensilelite/Tensile/TensileLogic/Run.py index a22eab2d22..faf0ae32de 100644 --- a/tensilelite/Tensile/TensileLogic/Run.py +++ b/tensilelite/Tensile/TensileLogic/Run.py @@ -1,62 +1,155 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + + import yaml import functools from pathlib import Path +from multiprocessing import Pool +from typing import List, Dict + +from Tensile.Common import ( + assignGlobalParameters, + ParallelMap2, + print1, + makeIsaInfoMap, + SUPPORTED_ISA, + IsaVersion, + IsaInfo, + verbosity +) +from Tensile.Common.GlobalParameters import globalParameters -from Tensile.Common import globalParameters, assignGlobalParameters, ParallelMap2 from Tensile.LibraryIO import readYAML from Tensile.Toolchain.Validators import validateToolchain +from Tensile.CustomKernels import isCustomKernelConfig, getCustomKernelConfig +from Tensile import CUSTOM_KERNEL_PATH from .ParseArguments import parseArguments from .ValidMatrixInstruction import validateMatrixInstruction +from .ValidWorkGroup import validateWorkGroup -def getParams(cxxCompiler): - gp = globalParameters +def handleCustomKernel(sol: dict, isaInfoMap: dict): + if not isCustomKernelConfig(sol): + return sol - gpcache = Path.cwd() / "gpcache.yaml" - if gpcache.exists(): - with open(gpcache, "r") as f: - gp = yaml.load(f, yaml.CSafeLoader) - else: - assignGlobalParameters({}, cxxCompiler) - with open(gpcache, "w") as f: - yaml.dump(gp, f, yaml.CSafeDumper) + name = sol["CustomKernelName"] + dir = CUSTOM_KERNEL_PATH + config = getCustomKernelConfig(name, {}, dir) + sol.update(config) + + mi = sol["MatrixInstruction"] + print1(f">> Found custom kernel: {name} with MI {mi}") + + if not (len(mi) == 4 or len(mi) == 0): + raise ValueError(f">> Error: Custom kernels should have matrix instruction of length 4, or none at all, not length {len(mi)}\n{name}") - return gp + return sol -def runChecks(logicPath, gp, file): - if "Experimental" in file.parts: - return 0, 0 +def runChecks(logicPath: str, isaInfoMap: Dict[IsaVersion, IsaInfo], files: List[Path]): + """ + Run checks on the given files. + Args: + logicPath: Path to the logic directory. + gp: Global parameters. + files: List of files to check. + + Returns: + Tuple of (keep, total) where keep is the number of solutions to keep and + total is the total number of solutions. + """ keep, total = 0, 0 - solutions = readYAML(file)[5] # Solutions are the 5th index - for s in solutions: - total += 1 - keep += validateMatrixInstruction(s, file.relative_to(logicPath), gp) - print(f">> {file.relative_to(logicPath)}") + for file in files: + if "Experimental" in file.parts: + return keep, total + + + solutions = readYAML(file)[5] # Solutions are the 5th index + + print1(f">> {file.relative_to(logicPath)}") + for s in solutions: + s = handleCustomKernel(s, isaInfoMap) + + if all( + [ + validateMatrixInstruction(s, isaInfoMap, file.relative_to(logicPath)), + validateWorkGroup(s, isaInfoMap, file.relative_to(logicPath)), + ] + ): + keep += 1 + total += 1 return keep, total def main(): args = parseArguments() + + global verbosity + verbosity = args.Verbose + + jobs = int(args.Jobs) cxxCompiler = validateToolchain(args.CxxCompiler) - gp = getParams(cxxCompiler) + + isaInfoMap = makeIsaInfoMap(SUPPORTED_ISA, cxxCompiler) + assignGlobalParameters({"PrintSolutionRejectionReason": True}, isaInfoMap) logicPath = Path(args.LogicPath) - pattern = "**/*.yaml" - files = logicPath.glob(pattern) - print(f"Checking logic files with glob {args.LogicPath}{pattern}") + if logicPath.is_file() and logicPath.suffix == ".yaml": + files = [logicPath] + else: + pattern = "**/*.yaml" + files = list(logicPath.glob(pattern)) - if not any([args.CheckMatrixInstruction]): - print("No checks specified. Exiting.") + if not any([args.Check]): + print1("No checks specified. Exiting.") exit(0) + if len(files) == 0: + print1(f"No files found in {logicPath}") + exit(1) - fn = functools.partial(runChecks, logicPath, gp) - results = ParallelMap2(fn, files, multiArg=False, procs=args.Jobs) + print1(f"Found {len(files)} files") - keep = sum([x[0] for x in results]) - total = sum([x[1] for x in results]) + batchSize = len(files) // min(len(files), jobs) + batches = (files[i : i + batchSize] for i in range(0, len(files), batchSize)) + + fn = functools.partial(runChecks, logicPath, isaInfoMap) + keep, total = 0, 0 + # with Pool(processes=jobs) as pool: + # results = pool.map_async(fn, batches) + + # # TIP: This is how to use joblib. Leave for reference. + results = ParallelMap2( + fn, batches, multiArg=False, procs=jobs, return_as="list" + ) + + for _keep, _total in results: + # for _keep, _total in results.get(): + keep += _keep + total += _total rejects = total - keep print(f"Total {total} solutions") diff --git a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py index 7d1ca2c926..7279716c6e 100644 --- a/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py +++ b/tensilelite/Tensile/TensileLogic/ValidMatrixInstruction.py @@ -1,118 +1,73 @@ -import math +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +""" +ValidMatrixInstruction +--- +Format: (M x N x K x B) + XDLOPS tile definition, only valid for gfx908, gfx90a + MxNxKxB specifies matrix instruction variants + MxNxB determines the shape of the C tile each instruction worked on + K determines the unroll depth + +Alternative format: (M x N x K x B x MIBlockM x WaveTileM x WaveTileN x WaveM x WaveN) + (Note: MxN means M-by-N in the following comments) + MIBlockM determines how many blocks along M dimension for multi-block MI variants. Concrete examples: + - MI 16x16x1x4 (4-block variant) with MIBlockM=4 -> (16x16)*(4x1)=64x16 tile per instruction executed + - MI 32x32x1x2 (2-block variant) with MIBlockM=1 -> (32x32)*(1x2)=32x64 tile per instruction executed + WaveTileM/N are dimensions of the C tile each wave works on, and is close to the concept of ThreadTile in classic VALU kernels + - WT 4x1 -> each wave executes 4x1 matrix instructions on the C tile of total area (4*MITileM)x(1*MITileN) + WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads + - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2 + Putting it all together: + - [32, 32, 1, 2, 1, 4, 1, 2, 2] + ^^^^^^^^^^^^ ^ ^^^^ ^^^^ + MatrixInst BlkM WT Wave + - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile + Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used + +Notes: + - If empty, do not use these instructions +""" + +from typing import Dict from pathlib import Path -from inspect import currentframe, getframeinfo -MI_KEY: str = "MatrixInstruction" -MI_ENABLED_KEY: str = "EnableMatrixInstruction" +from Tensile.SolutionStructs import reject +from Tensile.Common import IsaVersion, IsaInfo, print1, elineno +from Tensile.Common.Architectures import SUPPORTED_ISA +from Tensile.Common.ValidParameters import validMatrixInstructions, validMFMA, validWMMA, validSMFMA +from Tensile.TensileInstructions.DataType import DataType -validMFMA = {} -validMFMA["H"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] -validMFMA["S"] = [[32, 32, 1, 2], [32, 32, 2, 1], [16, 16, 1, 4], [16, 16, 4, 1], [4, 4, 1, 16]] -validMFMA["B"] = [[32, 32, 2, 2], [32, 32, 4, 1], [16, 16, 2, 4], [16, 16, 8, 1], [4, 4, 2, 16]] -validMFMA["4xi8"] = [ - [32, 32, 4, 2], - [32, 32, 8, 1], - [16, 16, 4, 4], - [16, 16, 16, 1], - [4, 4, 4, 16], - [32, 32, 16, 1], - [16, 16, 32, 1], -] -validMFMA["D"] = [[16, 16, 4, 1], [4, 4, 4, 4]] -validMFMA["B1k"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]] -validMFMA["C"] = validMFMA["S"] -validMFMA["Z"] = validMFMA["D"] -validMFMA["I8"] = [ - [32, 32, 4, 2], - [32, 32, 8, 1], - [16, 16, 4, 4], - [16, 16, 16, 1], - [4, 4, 4, 16], -] + [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["X"] = [[32, 32, 4, 1], [16, 16, 8, 1]] -validMFMA["F8"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["B8"] = validMFMA["F8"] -validMFMA["F8B8"] = validMFMA["F8"] -validMFMA["B8F8"] = validMFMA["F8"] -validMFMA["F8N"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validMFMA["B8N"] = validMFMA["F8N"] -validMFMA["F8B8N"] = validMFMA["F8N"] -validMFMA["B8F8N"] = validMFMA["F8N"] -validWMMA = [ - [16, 16, 16, 1], -] -validTT = 32 -validMFMA["_format9"] = [] - -for MFMA in [ - validMFMA["H"], - validMFMA["S"], - validMFMA["B"], - validMFMA["D"], - validMFMA["X"], - validMFMA["F8N"], - validWMMA, -]: - for MI in MFMA: - for bm in range(int(math.log(MI[3], 2)) + 1): - for tt0 in range(1, validTT + 1): - for tt1 in range(1, validTT + 1): - for wave_m in range(3): - for wave_n in range(3): - validMFMA["_format9"].append( - [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] - ) -validMatrixInstructions = ( - [[], [-1]] - + validMFMA["H"] - + validMFMA["S"] - + validMFMA["B"] - + validMFMA["D"] - + validMFMA["B1k"] - + validMFMA["X"] -) -validMatrixInstructions = validMatrixInstructions + validMFMA["_format9"] - -validSMFMA = {} -validSMFMA["H"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validSMFMA["B"] = [[32, 32, 16, 1], [16, 16, 32, 1]] -validSMFMA["4xi8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["I8"] = validSMFMA["4xi8"] -validSMFMA["F8"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["B8"] = validSMFMA["F8"] -validSMFMA["F8B8"] = validSMFMA["F8"] -validSMFMA["B8F8"] = validSMFMA["F8"] -validSMFMA["F8N"] = [[32, 32, 32, 1], [16, 16, 64, 1]] -validSMFMA["B8N"] = validSMFMA["F8N"] -validSMFMA["F8B8N"] = validSMFMA["F8N"] -validSMFMA["B8F8N"] = validSMFMA["F8N"] -validSMFMA["_format9"] = [] -for SMFMA in [validSMFMA["H"], validSMFMA["B"], validSMFMA["4xi8"], validSMFMA["F8N"]]: - for MI in SMFMA: - for bm in range(int(math.log(MI[3], 2)) + 1): - for tt0 in range(1, validTT + 1): - for tt1 in range(1, validTT + 1): - for wave_m in range(3): - for wave_n in range(3): - validSMFMA["_format9"].append( - [MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n] - ) -validSparseMatrixInstructions = validSMFMA["H"] + validSMFMA["B"] + validSMFMA["4xi8"] -validMatrixInstructions = ( - validMatrixInstructions + validSparseMatrixInstructions + validSMFMA["_format9"] -) - - -def elineno(): - """ - Return the file name and line number of the caller. - """ - frame = getframeinfo(currentframe().f_back) - return f"{Path(frame.filename).name}:{frame.lineno}" +MI_KEY: str = "MatrixInstruction" +MI_ENABLED_KEY: str = "EnableMatrixInstruction" -def validateMatrixInstruction(solution: dict, filepath: Path, params: dict): +def validateMatrixInstruction( + solution: dict, isaInfoMap: Dict[IsaVersion, IsaInfo], filepath: Path +) -> bool: """ Validates the matrix instruction configured in the given solution. @@ -137,97 +92,137 @@ def validateMatrixInstruction(solution: dict, filepath: Path, params: dict): AssertionError: If any of the validation checks fail. """ try: - _validateMatrixInstruction(solution, params) + validateMIParameters(solution, isaInfoMap) + assert solution["Valid"], f"Solution was rejected: {elineno()}" return True except AssertionError as e: - print(f"Validation failed: {filepath} (index {solution['SolutionIndex']})") - print(f"Error: file: {e}") + print( + f"Error: Validation failed: {e} (file: {filepath}, index: {solution['SolutionIndex']})" + ) return False -def _validateMatrixInstruction(solution: dict, params: dict): - """ - Function to validate the matrix instruction for the provided solution. - See exported function for more details. - """ - assert MI_KEY in solution, elineno() - assert MI_ENABLED_KEY in solution, elineno() - assert not (solution[MI_KEY] == [] and solution[MI_ENABLED_KEY] == True), elineno() - - isa = tuple(solution["ISA"]) - miFull = solution[MI_KEY] +def validateMIParameters( + solution: dict, isaInfoMap: Dict[IsaVersion, IsaInfo], printSolutionRejectionReason: bool = True +): + assert MI_KEY in solution, elineno() + ": missing MatrixInstruction" + assert MI_ENABLED_KEY in solution, elineno() + ": missing EnableMatrixInstruction" + assert not (solution[MI_KEY] == [] and solution[MI_ENABLED_KEY] == True), ( + elineno() + ": MI empty but enabled" + ) + + isa = IsaVersion(*solution["ISA"]) + assert isa in SUPPORTED_ISA, elineno() + ": Unsupported ISA: " + str(isa) + # TODO: Temporary until all 940/941 ISAs are removed + if (9, 4, 0) <= isa <= (9, 4, 1): + isa = (9, 4, 2) + + ptype = solution["ProblemType"] + isSparse = ptype.get("Sparse", 0) + miDataType = DataType( + ptype["DataType"] + if not solution.get("EnableF32XdlMathOp", False) + else ptype["F32XdlMathOp"] + ) + + mi4 = solution[MI_KEY] miEnabled = solution[MI_ENABLED_KEY] + assert len(mi4) == 4 or len(mi4) == 0, elineno() + ": MI length not 4 or 0" + if len(mi4) == 0: + assert miEnabled == False, elineno() + return - assert miFull in validMatrixInstructions, elineno() + assert solution["MatrixInstM"] == mi4[0] + assert solution["MatrixInstN"] == mi4[1] + assert solution["MatrixInstK"] == mi4[2] + assert solution["MatrixInstB"] == mi4[3] - if len(solution[MI_KEY]) == 9: - wfsize = solution["WavefrontSize"] - mi = [miFull[0], miFull[1], miFull[2], miFull[3]] - waves = miFull[7] * miFull[8] - miwg0 = miFull[4] * miFull[0] * miFull[7] # Matrix instruction work group 0 - miwg1 = waves * wfsize // miwg0 + assert mi4 in validMatrixInstructions, f"{elineno()} : invalid MI4: {str(mi4)} for type {miDataType.toChar()}" - isSparse = solution["ProblemType"]["Sparse"] - miDataType = ( - solution["ProblemType"]["DataType"] - if (not solution["EnableF32XdlMathOp"]) - else solution["ProblemType"]["F32XdlMathOp"] - ) - miBlock = solution["MIBlock"] - miWaveGroup = solution["MIWaveGroup"] - miWaveTile = solution["MIWaveTile"] - miInputPerThread = solution["MIInputPerThread"] - miInputPerThreadA = solution["MIInputPerThreadA"] - miInputPerThreadB = solution["MIInputPerThreadB"] - miInutPerThreadMeta = solution["MIInputPerThreadMetadata"] - - # Check work group - assert solution["WorkGroup"] == [miwg0, miwg1], elineno() - - # Check datatype - if not isSparse: - if params["AsmCaps"][isa]["HasMFMA"]: - if not (miDataType.toChar() in validMFMA and mi in validMFMA[miDataType.toChar()]): - assert miDataType.isBFloat16() and mi in validMFMA["B1k"], elineno() - elif params["AsmCaps"][isa]["HasWMMA"]: - assert mi in validWMMA, elineno() - else: - assert miDataType.toChar() in validSMFMA and mi in validSMFMA[miDataType.toChar()], elineno() - - if (not params["AsmCaps"][isa]["HasMFMA"]) and params["AsmCaps"][isa]["HasWMMA"]: - if isa[0] == 10 or isa[0] == 11: - assert miInputPerThread == mi[2], elineno() - - assert solution["MFMA_BF16_1K"] == False, elineno() - - # Check MIBlock - assert miBlock[0] == mi[0], elineno() - assert miBlock[1] == mi[1], elineno() - assert miBlock[2] == mi[2], elineno() - assert miBlock[3] == mi[3], elineno() - assert miBlock[4] == min(miwg0 // mi[0], mi[3]), elineno() - assert miBlock[5] == mi[3] // miBlock[4], elineno() - - # Check MIWaveGroup - assert miWaveGroup[0] == min((miwg0 // mi[0]) // miBlock[4], waves), elineno() - assert miWaveGroup[1] == waves // miWaveGroup[0], elineno() - - # Check MIWaveTile - assert miWaveTile[0] == mi[5], elineno() - assert miWaveTile[1] == mi[6], elineno() - - # Check MIInputPerThread - assert miInputPerThread == mi[0] * mi[2] * mi[3] // wfsize, elineno() - - # TODO: sparsity in hipBLASLt appears to be unused or always zero - sparseA = not isSparse if isSparse != 2 else False - sparseB = isSparse == 2 if isSparse else False - assert miInputPerThreadA == miInputPerThread if not sparseA else miInputPerThread // 2, elineno() - assert miInputPerThreadB == miInputPerThread if not sparseB else miInputPerThread // 2, elineno() - assert miInutPerThreadMeta == miInputPerThread if not isSparse else miInputPerThread // 8, elineno() - - assert miEnabled == True, elineno() - elif miFull != [] and len(miFull) == 4: - assert miEnabled == True, elineno() + mi9 = [mi4[0], mi4[1], mi4[2], mi4[3]] + assert "MatrixInstBM" in solution, elineno() + ": missing MatrixInstBM" + mi9.append(solution["MatrixInstBM"]) + assert "MIWaveTile" in solution, elineno() + ": missing MIWaveTile" + mi9.extend(solution["MIWaveTile"]) + assert "MIWaveGroup" in solution, elineno() + ": missing MIWaveGroup" + mi9.extend(solution["MIWaveGroup"]) + + assert len(mi4) == 4 and len(mi9) == 9, elineno() + " MI4: " + str(mi4) + " MI9: " + str(mi9) + + if not miEnabled: + return False + + + wfsize = solution["WavefrontSize"] + waves = solution["MIWaveGroup"][0] * solution["MIWaveGroup"][1] + wg0 = mi9[4] * mi9[0] * mi9[7] # Work group 0 + + hasMFMA = isaInfoMap[isa].asmCaps["HasMFMA"] + hasWMMA = isaInfoMap[isa].asmCaps["HasWMMA"] + + miBlock = solution["MIBlock"] + miWaveGroup = solution["MIWaveGroup"] + miWaveTile = solution["MIWaveTile"] + + # Check datatype + if not isSparse: # If it's sparse + if hasMFMA: # and it supports MFMA + # but is invalid MFMA + if not miDataType.toChar() in validMFMA and mi4 in validMFMA[miDataType.toChar()]: + if miDataType.isBFloat16() and mi4 in validMFMA["B1k"]: # but is valid bf16 MFMA + assert solution["MFMA_BF16_1K"], elineno() + else: + return not reject( + solution, + True,#printSolutionRejectionReason, + f"Invalid MFMA BFloat16 configuration: {solution}", + ) + elif hasWMMA and (not mi4 in validWMMA): + return not reject( + solution, printSolutionRejectionReason, f"Invalid WMMA configuration: {solution}" + ) else: - assert miEnabled == False, elineno() + if not (miDataType.toChar() in validSMFMA and mi4 in validSMFMA[miDataType.toChar()]): + return not reject( + solution, printSolutionRejectionReason, f"Invalid SMFMA configuration: {solution}" + ) + + # Check MIBlock + assert miBlock[0] == mi4[0], elineno() + assert miBlock[1] == mi4[1], elineno() + assert miBlock[2] == mi4[2], elineno() + assert miBlock[3] == mi4[3], elineno() + assert miBlock[4] == min(wg0 // mi4[0], mi4[3]), elineno() + assert miBlock[5] == mi4[3] // miBlock[4], elineno() + + # Check MIWaveGroup + assert miWaveGroup[0] == min((wg0 // mi4[0]) // miBlock[4], waves), elineno() + assert miWaveGroup[1] == waves // miWaveGroup[0], elineno() + + # Check MIWaveTile + assert miWaveTile[0] == mi9[5], elineno() + assert miWaveTile[1] == mi9[6], elineno() + + # Check MIInputPerThread + miInputPerThread = solution["MIInputPerThread"] + + if (not hasMFMA) and hasWMMA: + if isa[0] == 10 or isa[0] == 11: + assert miInputPerThread == mi4[2], elineno() + + # If Navi architecture, the input per thread is different + if IsaVersion(10, 0, 0) <= isa <= IsaVersion(11, 0, 2): + assert miInputPerThread == mi4[2], elineno() + else: + assert miInputPerThread == mi4[0] * mi4[2] * mi4[3] // wfsize, f"{elineno()} MIInputPerThread: {miInputPerThread} != {mi4[0]} * {mi4[2]} * {mi4[3]} / {wfsize} = {mi4[0] * mi4[2] * mi4[3] // wfsize}" + + + # miInputPerThreadA = solution["MIInputPerThreadA"] + # miInputPerThreadB = solution["MIInputPerThreadB"] + # miInutPerThreadMeta = solution["MIInputPerThreadMetadata"] + # sparseA = not isSparse if isSparse != 2 else False + # sparseB = isSparse == 2 if isSparse else False + # assert miInputPerThreadA == miInputPerThread if not sparseA else miInputPerThread // 2, elineno() + # assert miInputPerThreadB == miInputPerThread if not sparseB else miInputPerThread // 2, elineno() + # assert miInutPerThreadMeta == miInputPerThread if not isSparse else miInputPerThread // 8, elineno() + return True diff --git a/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py b/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py new file mode 100644 index 0000000000..da4dca47e0 --- /dev/null +++ b/tensilelite/Tensile/TensileLogic/ValidWorkGroup.py @@ -0,0 +1,52 @@ +################################################################################ +# +# Copyright (C) 2025 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +################################################################################ + +""" +ValidWorkGroup +--- +Dimensions of the workgroup which will operate on a tile and share lds +Example: ( wg0 x wg1 x LocalSplitU ) +""" + +from typing import Dict + +from Tensile.Common import IsaVersion, IsaInfo, elineno +from Tensile.Common.ValidParameters import validWorkGroups + + +def validateWorkGroup(solution: dict, isaInfoMap: Dict[IsaVersion, IsaInfo], filepath: str): + try: + _validateWorkGroup(solution, isaInfoMap) + assert solution["Valid"], f"Solution was rejected: {elineno()}" + return True + except AssertionError as e: + print( + f"Error: Validation failed: {e} (file: {filepath}, index: {solution['SolutionIndex']})" + ) + return False + + +def _validateWorkGroup(solution: dict, isaInfoMap: dict): + assert "WorkGroup" in solution, elineno() + assert solution["WorkGroup"] in validWorkGroups, elineno() diff --git a/tensilelite/Tensile/Tests/conftest.py b/tensilelite/Tensile/Tests/conftest.py index 894510feef..975ed7e6d6 100644 --- a/tensilelite/Tensile/Tests/conftest.py +++ b/tensilelite/Tensile/Tests/conftest.py @@ -139,8 +139,8 @@ def __enter__(self): Common.restoreDefaultGlobalParameters() if args.CxxCompiler: Common.globalParameters["CxxCompiler"] = args.CxxCompiler - - Common.assignGlobalParameters({}) + isa = Common.detectGlobalCurrentISA(args.device) + Common.assignGlobalParameters({}, isa) overrideParameters = Tensile.argUpdatedGlobalParameters(args) for key, value in overrideParameters.items(): diff --git a/tensilelite/Tensile/Tests/unit/testCustomKernelConfig.py b/tensilelite/Tensile/Tests/unit/testCustomKernelConfig.py new file mode 100644 index 0000000000..9924f4b9af --- /dev/null +++ b/tensilelite/Tensile/Tests/unit/testCustomKernelConfig.py @@ -0,0 +1,165 @@ +import pytest +import yaml +from pprint import pformat + +from Tensile.CustomKernels import getCustomKernelConfig +from Tensile.SolutionStructs import matrixInstructionToMIParameters +from Tensile.Common.Architectures import SUPPORTED_ISA +from Tensile.Common.Capabilities import makeIsaInfoMap +from Tensile.Common.Types import IsaVersion +from Tensile.Common.GlobalParameters import defaultSolution +from Tensile.Toolchain.Validators import validateToolchain +from Tensile.TensileLogic.ValidMatrixInstruction import validateMIParameters + +cxxCompiler = validateToolchain("amdclang++") + +ISA_INFO_MAP = makeIsaInfoMap(SUPPORTED_ISA, cxxCompiler) + + +# @pytest.mark.parametrize("objs", [("TestKernel", testKernelDir, configResult)]) +def testConvert9ItemCustomKernelConfig(): + + inputConf = yaml.load( + """ +custom.config: + ProblemType: + OperationType: GEMM + DataTypeA: f8n + DataTypeB: h + UseScaleAB: "Scalar" + DataType: h + DestDataType: s + ComputeDataType: s + HighPrecisionAccumulate: True + TransposeA: False + TransposeB: False + UseBias: 1 + Activation: True + UseScaleAlphaVec: 1 + UseBeta: True + Batched: True + GroupedGemm: True + SupportUserArgs: True + MatrixInstruction: [32, 32, 8, 1, 5, 6, 7, 8, 9] + 1LDSBuffer: 1 + ScheduleIterAlg: 3 + DepthU: 32 + StaggerU: 0 + WorkGroupMapping: 8 + WaveSeparateGlobalReadA: 1 + WaveSeparateGlobalReadB: 1 + GlobalReadVectorWidthA: 4 + GlobalReadVectorWidthB: 2 + AssertFree0ElementMultiple: 4 + AssertSummationElementMultiple: 1 + NoReject: 1 + InternalSupportParams: + KernArgsVersion: 0 + SupportUserGSU: False + SupportCustomWGM: False + SupportCustomStaggerU: False + UseUniversalArgs: False +""", + yaml.SafeLoader, + ) + inputConf = inputConf["custom.config"] + + isa = IsaVersion(9, 4, 2) + wavefrontSize = 48 + workGroup = [4, 5, 6] + + outputConf = matrixInstructionToMIParameters( + inputConf["MatrixInstruction"], + isa, + wavefrontSize, + inputConf["ProblemType"], + workGroup, + ISA_INFO_MAP, + ) + + input = { + "MatrixInstruction": inputConf["MatrixInstruction"], + } + + print("inputConf: ", pformat(input)) + print("outputConf: ", pformat(outputConf)) + + assert outputConf["MatrixInstruction"] == [32, 32, 8, 1] + assert outputConf["EnableMatrixInstruction"] == True + assert outputConf["MIBlock"] == [32, 32, 8, 1, 1, 1] + assert outputConf["MIWaveGroup"] == [40, 1] + assert outputConf["MIWaveTile"] == [6, 7] + assert outputConf["MatrixInstBM"] == 1 + assert outputConf["MatrixInstBN"] == 1 + assert outputConf["MIInputPerThread"] == 5 + assert outputConf["MIInputPerThreadA"] == 5 + assert outputConf["MIInputPerThreadB"] == 5 + assert outputConf["MIInputPerThreadMetadata"] == 5 + assert outputConf["ThreadTile"] == [1, 1] + assert outputConf["Sparse"] == 0 + assert outputConf["WorkGroup"] == [1280, 2, 6] # Why do we change the workgroup here? + assert outputConf["WavefrontSize"] == 48 + assert outputConf["ISA"] == isa + assert outputConf["EnableF32XdlMathOp"] == False + assert outputConf["MFMA_BF16_1K"] == False + + solution = defaultSolution + solution.update(inputConf) + solution.update(outputConf) + + assert validateMIParameters(solution, ISA_INFO_MAP, True) == True + +""" +def testConvert4ItemCustomKernelConfig(): + + inputConf = yaml.load("1234", + yaml.SafeLoader, + ) + inputConf = inputConf["custom.config"] + + isa = IsaVersion(9, 4, 2) + wavefrontSize = 48 + workGroup = [4, 5, 6] + + outputConf = matrixInstructionToMIParameters( + inputConf["MatrixInstruction"], + isa, + wavefrontSize, + inputConf["ProblemType"], + workGroup, + ISA_INFO_MAP, + ) + + input = { + "MatrixInstruction": inputConf["MatrixInstruction"], + } + + print("inputConf: ", pformat(input)) + print("outputConf: ", pformat(outputConf)) + + assert outputConf["MatrixInstruction"] == [32, 32, 8, 1] + assert outputConf["EnableMatrixInstruction"] == True + assert outputConf["MIBlock"] == [32, 32, 8, 1, 1, 1] + assert outputConf["MIWaveGroup"] == [40, 1] + assert outputConf["MIWaveTile"] == [6, 7] + assert outputConf["MatrixInstBM"] == 1 + assert outputConf["MatrixInstBN"] == 1 + assert outputConf["MIInputPerThread"] == 5 + assert outputConf["MIInputPerThreadA"] == 5 + assert outputConf["MIInputPerThreadB"] == 5 + assert outputConf["MIInputPerThreadMetadata"] == 5 + assert outputConf["ThreadTile"] == [1, 1] + assert outputConf["Sparse"] == 0 + assert outputConf["WorkGroup"] == [1280, 2, 6] # Why do we change the workgroup here? + assert outputConf["WavefrontSize"] == 48 + assert outputConf["ISA"] == isa + assert outputConf["EnableF32XdlMathOp"] == False + assert outputConf["MFMA_BF16_1K"] == False + + solution = defaultSolution + solution.update(inputConf) + solution.update(outputConf) + + assert validateMIParameters(solution, ISA_INFO_MAP, True) == True + +""" diff --git a/tensilelite/Tensile/Toolchain/Assembly.py b/tensilelite/Tensile/Toolchain/Assembly.py index 99a9ed8512..1c38e94e79 100644 --- a/tensilelite/Tensile/Toolchain/Assembly.py +++ b/tensilelite/Tensile/Toolchain/Assembly.py @@ -24,129 +24,31 @@ import collections import math -import os -import shlex import shutil import subprocess from pathlib import Path -from typing import List, Union +from typing import List, Union, NamedTuple -from ..Common import globalParameters, print2, ensurePath, SemanticVersion, isaToGfx -from ..KernelWriterAssembly import KernelWriterAssembly -from ..Toolchain.Validators import getVersion +from ..Common import print2, isaToGfx, print1 from ..SolutionStructs import Solution -class AssemblyToolchain: - def __init__(self, assembler: str, bundler: str, buildIdKind: str, coVersion: str): - self.assembler = assembler - self.assemblerVersion = SemanticVersion(*[int(c) for c in getVersion(assembler).split(".")[:3]]) - self.bundler = bundler - self.buildIdKind = buildIdKind - self.coVersion = coVersion - - def invoke(self, args: List[str], desc: str=""): - """Invokes a subprocess with the provided arguments. - - Args: - args: A list of arguments to pass to the subprocess. - desc: A description of the subprocess invocation. - - Raises: - RuntimeError: If the subprocess invocation fails. - """ - print2(f"{desc}: {' '.join(args)}") - try: - out = subprocess.check_output(args, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as err: - raise RuntimeError( - f"Error with {desc}: {err.output}\n" - f"Failed command: {' '.join(args)}" - ) - print2(f"Output: {out}") - return out - - def assemble(self, srcPath: str, destPath: str, gfx: str, wavefrontSize: int, debug: bool=False): - """Assemble an assembly source file into an object file. - - Args: - srcPath: The path to the assembly source file. - destPath: The destination path for the generated object file. - coVersion: The code object version to use. - isa: The target GPU architecture in ISA format. - wavefrontSize: The wavefront size to use. - """ - launcher = shlex.split(os.environ.get('Tensile_ASM_COMPILER_LAUNCHER', '')) - args = [ - *launcher, - self.assembler, - "-x", "assembler", - "--target=amdgcn-amd-amdhsa", - f"-mcode-object-version={self.coVersion}", - f"-mcpu={gfx}", - "-mwavefrontsize64" if wavefrontSize == 64 else "-mno-wavefrontsize64" - "-g" if debug else "", - "-c", - "-o", destPath, srcPath - ] - - return self.invoke(args, "Assembling assembly source code into object file (.s -> .o)") - - def link(self, srcPaths: List[str], destPath: str): - """Links object files into a code object file. - - Args: - srcPaths: A list of paths to object files. - destPath: A destination path for the generated code object file. - - Raises: - RuntimeError: If linker invocation fails. - """ - if os.name == "nt": - # Use args file on Windows b/c the command may exceed the limit of 8191 characters - with open(Path.cwd() / "clang_args.txt", "wt") as file: - file.write(" ".join(objFiles)) - file.flush() - args = [ - self.assembler, - "--target=amdgcn-amd-amdhsa", - "-o", destPath, "@clang_args.txt"] - else: - args = [ - self.assembler, - "--target=amdgcn-amd-amdhsa", - "-Xlinker", f"--build-id={self.buildIdKind}", - "-o", destPath, *srcPaths - ] - - return self.invoke(args, "Linking assembly object files into code object (*.o -> .co)") - - def compress(self, srcPath: str, destPath: str, gfx: str): - """Compresses a code object file using the provided bundler. - - Args: - srcPath: The source path of the code object file to be compressed. - destPath: The destination path for the compressed code object file. - gfx: The target GPU architecture. - - Raises: - RuntimeError: If compressing the code object file fails. - """ - args = [ - self.bundler, - "--compress", - "--type=o", - "--bundle-align=4096", - f"--targets=host-x86_64-unknown-linux-gnu,hipv4-amdgcn-amd-amdhsa-unknown-{gfx}", - "--input=/dev/null", - f"--input={srcPath}", - f"--output={destPath}", - ] - - return self.invoke(args, "Bundling/compressing code object file (.co -> .co)") - - -def _batchObjectFiles(objFiles: List[str], coPathDest: Union[Path, str], maxObjFiles: int=10000) -> List[str]: +from .Component import Assembler, Linker, Bundler + +class AssemblyToolchain(NamedTuple): + assembler: Assembler + linker: Linker + bundler: Bundler + + +def makeAssemblyToolchain(assembler_path, bundler_path, co_version, build_id_kind="sha1"): + compiler = Assembler(assembler_path, co_version) + linker = Linker(assembler_path, build_id_kind) + bundler = Bundler(bundler_path) + return AssemblyToolchain(compiler, linker, bundler) + + +def _batchObjectFiles(ldPath: str, objFiles: List[str], coPathDest: Union[Path, str], maxObjFiles: int=10000) -> List[str]: numObjFiles = len(objFiles) if numObjFiles <= maxObjFiles: @@ -160,7 +62,7 @@ def _batchObjectFiles(objFiles: List[str], coPathDest: Union[Path, str], maxObjF for batch, filename in zip(batchedObjFiles, newObjFiles): if len(batch) > 1: - args = [globalParameters["ROCmLdPath"], "-r"] + batch + [ "-o", filename] + args = [ldPath, "-r"] + batch + [ "-o", filename] print2(f"Linking object files into fewer object files: {' '.join(args)}") subprocess.check_call(args) newObjFilesOutput.append(filename) @@ -169,13 +71,15 @@ def _batchObjectFiles(objFiles: List[str], coPathDest: Union[Path, str], maxObjF return newObjFilesOutput + def buildAssemblyCodeObjectFiles( - toolchain: AssemblyToolchain, + linker: Linker, + bundler: Bundler, + ldPath: str, kernels: List[Solution], - writer: KernelWriterAssembly, destDir: Union[Path, str], asmDir: Union[Path, str], - compress: bool=True + compress: bool=True, ): """Builds code object files from assembly files @@ -188,17 +92,12 @@ def buildAssemblyCodeObjectFiles( compress: Whether to compress the code object files. """ - isAsm = lambda k: k["KernelLanguage"] == "Assembly" - extObj = ".o" extCo = ".co" extCoRaw = ".co.raw" - destDir = Path(ensurePath(destDir)) - asmDir = Path(ensurePath(asmDir)) - archKernelMap = collections.defaultdict(list) - for k in filter(isAsm, kernels): + for k in kernels: archKernelMap[tuple(k['ISA'])].append(k) coFiles = [] @@ -208,20 +107,21 @@ def buildAssemblyCodeObjectFiles( gfx = isaToGfx(arch) - objectFiles = [str(asmDir / (writer.getKernelFileBase(k) + extObj)) for k in archKernels if 'codeObjectFile' not in k] + objectFiles = [str(asmDir / (k["BaseName"] + extObj)) for k in archKernels if 'codeObjectFile' not in k] coFileMap = collections.defaultdict(list) if len(objectFiles): coFileMap[asmDir / ("TensileLibrary_"+ gfx + extCoRaw)] = objectFiles for kernel in archKernels: coName = kernel.get("codeObjectFile", None) if coName: - coFileMap[asmDir / (coName + extCoRaw)].append(str(asmDir / (writer.getKernelFileBase(kernel) + extObj))) + coFileMap[asmDir / (coName + extCoRaw)].append(str(asmDir / (kernel["BaseName"] + extObj))) + for coFileRaw, objFiles in coFileMap.items(): - objFiles = _batchObjectFiles(objFiles, coFileRaw) - toolchain.link(objFiles, str(coFileRaw)) + objFiles = _batchObjectFiles(ldPath, objFiles, coFileRaw) + linker(objFiles, str(coFileRaw)) coFile = destDir / coFileRaw.name.replace(extCoRaw, extCo) if compress: - toolchain.compress(str(coFileRaw), str(coFile), gfx) + bundler.compress(str(coFileRaw), str(coFile), gfx) else: shutil.move(coFileRaw, coFile) coFiles.append(coFile) diff --git a/tensilelite/Tensile/Toolchain/Component.py b/tensilelite/Tensile/Toolchain/Component.py new file mode 100644 index 0000000000..6d5e40edbd --- /dev/null +++ b/tensilelite/Tensile/Toolchain/Component.py @@ -0,0 +1,342 @@ + +from os import name as os_name +from os import environ +from pathlib import Path +from re import search, IGNORECASE +from shlex import split +from subprocess import check_output, STDOUT, CalledProcessError, PIPE, run +from typing import List + +from Tensile.Common import SemanticVersion, print1 +from .Validators import ToolchainDefaults + +def _invoke(args: List[str], desc: str=""): + """Invokes a command with the provided arguments in a subprocess. + Args: + args: A list of arguments to pass to the subprocess. + desc: A description of the subprocess invocation. + Raises: + RuntimeError: If the subprocess invocation fails. + Return: + subprocess output + """ + #print1(f"{desc}: {' '.join(args)}") + try: + out = check_output(args, stderr=STDOUT) + except CalledProcessError as err: + raise RuntimeError( + f"Error with {desc}: {err.output}\n" + f"Failed command: {' '.join(args)}" + ) + #print2(f"Output: {out}") + return out + + +def _getVersion(executable: str, versionFlag: str, regex: str) -> str: + """Compute the version string of a toolchain component. + + Args: + executable: The toolchain component to check the version of. + versionFlag: The flag to pass to the executable to get the version. + regex: pattern used to extract version string. + Raises: + RuntimeError: If querying executable for version fails. + Return: + Executable version + """ + args = f'"{executable}" "{versionFlag}"' + try: + output = run(args, stdout=PIPE, shell=True).stdout.decode().strip() + match = search(regex, output, IGNORECASE) + result = match.group(1) if match else "" + return SemanticVersion(*[int(c.split("-")[0]) for c in result.split(".")[:3]]) + except Exception as e: + raise RuntimeError(f"Failed to get version when calling {args}: {e}") + + +def get_rocm_version() -> str: + """Compute the ROCm version string using hipconfig. + + Raises: + RuntimeError: If hipconfig fails to execute. + Return: + ROCm version string + """ + return _getVersion(ToolchainDefaults.HIP_CONFIG, "--version", r'(.+)') + + +class Component: + """A class used to represent a ROCm toolchain component such as clang++""" + + _rocm_version = get_rocm_version() + + def __init__(self, component_path: Path, version_flag: str="--version", version_regex: str=r"version\s+([\d.]+)"): + self._version = _getVersion(str(component_path), version_flag, version_regex) + self._component_path = component_path + + def __str__(self): + result = f"ROCm {Component._rocm_version.major}.{Component._rocm_version.minor}.{Component._rocm_version.patch} " + result += f"Component path: {self._component_path} version: {self._version.major}.{self._version.minor}.{self._version.patch}" + return result + + @property + def path(self): + return self._component_path + + @property + def version(self): + return self._version + + @property + def rocm_version(self): + return Component._rocm_version + + +class Assembler(Component): + """ + ROCm assembler class used to build objects from assembly source files. + + ... + + Attributes + ---------- + version : str + the version of the component + rocm_version : str + the ROCm version + path : str + path to assembler + + Methods + ------- + __call__(self, targetGfx: str, wavefrontSize: int, debug: bool, srcPath: str, destPath: str) + Invokes the assembler on the provided arguments + """ + + def __init__(self, component_path: Path, co_version: str, debug: bool=False): + """Constructs instance of assmebler. + + Args: + assembler_path: The path to the assember. + co_version: The code object version to use. + """ + + super(Assembler, self).__init__(component_path) + self._code_object_version = co_version + + self._default_args = [ + *split(environ.get('Tensile_ASM_COMPILER_LAUNCHER', '')), + str(component_path), + "-x", "assembler", + "--target=amdgcn-amd-amdhsa", + "-g" if debug else "", + f"-mcode-object-version={co_version}", + "-c", + ] + + def __call__(self, targetGfx: str, wavefrontSize: int, srcPath: str, destPath: str): + """Assemble an assembly source file into an object file. + Args: + targetGfx: The target GPU gfx architecture. + wavefrontSize: The wavefront size to use. + debug: add debug flags if True. + srcPath: The path to the assembly source file. + destPath: The destination path for the generated object file. + """ + args = self._default_args + args = [ + *args, + f"-mcpu={targetGfx}", + "-mwavefrontsize64" if wavefrontSize == 64 else "-mno-wavefrontsize64", + srcPath, + "-o", + destPath + ] + return _invoke(args, "Assembling assembly source code into object file (.s -> .o)") + + @property + def code_object_version(self): + return self._code_object_version + +class Compiler(Component): + """ + ROCm compiler class used to build objects from C++ source files. + + ... + + Attributes + ---------- + version : str + the version of the component + rocm_version : str + the ROCm version + path : str + path to compiler + + Methods + ------- + __call__(self, include_path: str, target_list: List[str], srcPath: str, destPath: str): + Invokes the compiler on the provided arguments + """ + + def __init__(self, compiler_path: Path, build_id_kind: str, asan_build: bool=False, save_temps: bool=False): + """Constructs and instance of a Compiler.""" + super(Compiler, self).__init__(compiler_path) + + self.default_args = [ + *split(environ.get("Tensile_CXX_COMPILER_LAUNCHER", "")), + compiler_path, + "-D__HIP_HCC_COMPAT_MODE__=1", + "--offload-device-only", + "-x", "hip", "-O3", + "-Xoffload-linker", f"--build-id={build_id_kind}", + "-std=c++17", + ] + + if asan_build: + self.default_args.extend(["-fsanitize=address", "-shared-libasan", "-fuse-ld=lld"]) + if save_temps: + self.default_args.append("--save-temps") + if os_name == "nt": # should we use fPIIC on all arches? + self.default_args.extend(["-fms-extensions", "-fms-compatibility", "-fPIC", "-Wno-deprecated-declarations"]) + + + def __call__(self, include_path: str, target_list: List[str], srcPath: str, destPath: str): + """Compiles a source file into an object file. + + Args: + include_path: Path appened to "-I" to directory with required include files. + target_list: List of offload architectures of the form gfxXYZ e.g. gfx942. + sercPath: The path to the source file. + destPath: Path to the object file output during compilation. + Raises: + RuntimeError: If the compilation command fails. + """ + archFlags = [f"--offload-arch={gfx}" for gfx in target_list] + args = [ + *(self.default_args), "-I", include_path, *archFlags, srcPath, "-c", "-o", destPath + ] + return _invoke(args, f"Compiling HIP source kernels into objects (.cpp -> .o)") + + +class Bundler(Component): + """ + ROCm bundler class used to unbundle objects into code object files. + + ... + + Attributes + ---------- + version : str + the version of the component + rocm_version : str + the ROCm version + Methods + ------- + __call__(self, targetGfx: str, wavefrontSize: int, debug: bool, srcPath: str, destPath: str) + Invokes the assembler on the provided arguments + def targets(self, objFile: str): + returns a list of target triple strings of the form amdgcn-amd--gfx942 + def compress(self, srcPath: str, destPath: str, target: str): + Compresses a code object file using the provided bundler. + """ + + def __init__(self, bundler_path: Path): + """Constructs and instance of a Bunder.""" + super(Bundler, self).__init__(bundler_path) + + def targets(self, objFile: str): + """returns a list of target triple strings of the form amdgcn-amd--gfx942""" + args = [self._component_path, "--type=o", f"--input={objFile}", "-list"] + return _invoke(args, f"Listing target triples in object file").decode().split("\n") + + def compress(self, srcPath: str, destPath: str, target: str): + """Compresses a code object file using the provided bundler. + + Args: + srcPath: The source path of the code object file to be compressed. + destPath: The destination path for the compressed code object file. + gfx: The target GPU architecture. + + Raises: + RuntimeError: If compressing the code object file fails. + """ + args = [ + self._component_path, + "--compress", + "--type=o", + "--bundle-align=4096", + f"--targets=host-x86_64-unknown-linux-gnu,hipv4-amdgcn-amd-amdhsa-unknown-{target}", + "--input=/dev/null", + f"--input={srcPath}", + f"--output={destPath}", + ] + + return _invoke(args, "Bundling/compressing code object file (.co -> .co)") + + def __call__(self, target: str, srcPath: str, destPath: str): + """Unbundles source code object files using the Clang Offload Bundler. + Args: + target: The target triple, see https://llvm.org/docs/AMDGPUUsage.html#target-triples. + srcPath: The path to the input object file. + destPath: The path to the unbundled code object. + Raises: + RuntimeError: If unbundling the source code object file fails. + """ + args = [ + self._component_path, + "--type=o", + f"--targets={target}", + f"--input={srcPath}", + f"--output={destPath}", + "--unbundle", + ] + return _invoke(args, f"Unbundling source code object file") + + +class Linker(Component): + """ + ROCm Linker class used to link objects into code object files. + + ... + + Attributes + ---------- + version : str + the version of the component + rocm_version : str + the ROCm version + Methods + ------- + __call__(self, srcPaths: List[str], destPath: str): + Invokes the linker on the provided arguments + """ + + def __init__(self, linker_path: Path, build_id_kind: str): + """Constructs and instance of a Linker.""" + super(Linker, self).__init__(linker_path) + self.default_args = [ + self._component_path, + "--target=amdgcn-amd-amdhsa", + "-Xlinker", f"--build-id={build_id_kind}", + ] + + + def __call__(self, srcPaths: List[str], destPath: str): + """Links object files into a code object file. + + Args: + srcPaths: A list of paths to object files. + destPath: A destination path for the generated code object file. + Raises: + RuntimeError: If linker invocation fails. + """ + if os_name == "nt": + # Use args file on Windows b/c the command may exceed the limit of 8191 characters + with open(Path.cwd() / "clang_args.txt", "wt") as file: + file.write(" ".join(srcPaths)) + file.flush() + args = [*(self.default_args), "-o", destPath, "@clang_args.txt"] + else: + args = [*(self.default_args), *srcPaths, "-o", destPath] + return _invoke(args, "Linking assembly object files into code object (*.o -> .co)") diff --git a/tensilelite/Tensile/Toolchain/Source.py b/tensilelite/Tensile/Toolchain/Source.py index 59ee225071..3cd54c5b35 100644 --- a/tensilelite/Tensile/Toolchain/Source.py +++ b/tensilelite/Tensile/Toolchain/Source.py @@ -24,118 +24,25 @@ import os import re -import shlex import shutil -import subprocess from pathlib import Path from timeit import default_timer as timer -from typing import List, Union - -from ..Common import globalParameters, print1, print2, ensurePath, splitArchs - -class SourceToolchain: - def __init__(self, compiler: str, bundler: str, buildIdKind: str, asanBuild: bool=False, saveTemps: bool=False): - self.compiler = compiler - self.bundler = bundler - self.buildIdKind = buildIdKind - self.asanBuild = asanBuild - self.saveTemps = saveTemps - - def invoke(self, args: List[str], desc: str=""): - """Invokes a subprocess with the provided arguments. - - Args: - args: A list of arguments to pass to the subprocess. - desc: A description of the subprocess invocation. - - Raises: - RuntimeError: If the subprocess invocation fails. - """ - print2(f"{desc}: {' '.join(args)}") - try: - out = subprocess.check_output(args, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as err: - raise RuntimeError( - f"Error with {desc}: {err.output}\n" - f"Failed command: {' '.join(args)}" - ) - print2(f"Output: {out}") - return out - - def compile(self, srcPath: str, destPath: str, includePath: str, gfxs: List[str]): - """Compiles a source file into an object file. - - Args: - cmdlineArchs: List of architectures for offloading. - kernelFile: The path to the kernel source file. - buildPath: The build directory path. - objectFilename: The name of the output object file. - outputPath: The output directory path. - globalParameters: A dictionary of global parameters. - - Raises: - RuntimeError: If the compilation command fails. - """ - launcher = shlex.split(os.environ.get("Tensile_CXX_COMPILER_LAUNCHER", "")) - - hipFlags = [ - "-D__HIP_HCC_COMPAT_MODE__=1", - "--offload-device-only", - "-x", "hip", "-O3", - "-I", includePath, - "-Xoffload-linker", f"--build-id={self.buildIdKind}", - "-std=c++17", - ] - if self.asanBuild: - hipFlags.extend(["-fsanitize=address", "-shared-libasan", "-fuse-ld=lld"]) - if self.saveTemps: - hipFlags.append("--save-temps") - if os.name == "nt": - hipFlags.extend(["-fms-extensions", "-fms-compatibility", "-fPIC", "-Wno-deprecated-declarations"]) - - archFlags = [f"--offload-arch={gfx}" for gfx in gfxs] - - args = [ - *launcher, self.compiler, *hipFlags, *archFlags, srcPath, "-c", "-o", destPath - ] - - return self.invoke(args, f"Compiling HIP source kernels into objects (.cpp -> .o)") - - - def targets(self, objFile: str): - """Lists the target triples in an object file. - - Args: - objFile: The object file path. - - Returns: - List of target triples in the object file. - """ - args = [self.bundler, "--type=o", f"--input={objFile}", "-list"] - return self.invoke(args, f"Listing target triples in object file").decode().split("\n") - - def unbundle(self, target: str, srcPath: str, destPath: str): - """Unbundles source code object files using the Clang Offload Bundler. - - Args: - target: The target triple, see https://llvm.org/docs/AMDGPUUsage.html#target-triples. - infile: The path to the input object file. - outfileRaw: The path to the unbundled code object. - - Raises: - RuntimeError: If unbundling the source code object file fails. - """ - args = [ - self.bundler, - "--type=o", - f"--targets={target}", - f"--input={srcPath}", - f"--output={destPath}", - "--unbundle", - ] - - return self.invoke(args, f"Unbundling source code object file") +from typing import List, Union, NamedTuple + +from ..Common import print1, ensurePath + +from .Component import Compiler, Bundler + +class SourceToolchain(NamedTuple): + compiler: Compiler + bundler: Bundler + + +def makeSourceToolchain(compiler_path, bundler_path, asan_build=False, build_id_kind="sha1", save_temps=False): + compiler = Compiler(compiler_path, build_id_kind, asan_build, save_temps) + bundler = Bundler(bundler_path) + return SourceToolchain(compiler, bundler) def _computeSourceCodeObjectFilename(target: str, base: str, buildPath: Union[Path, str], arch: str) -> Union[Path, None]: @@ -165,12 +72,13 @@ def _computeSourceCodeObjectFilename(target: str, base: str, buildPath: Union[Pa def buildSourceCodeObjectFiles( - toolchain: SourceToolchain, + compiler: Compiler, + bundler: Bundler, destDir: Union[Path, str], tmpObjDir: Union[Path, str], includeDir: Union[Path, str], kernelPath: Union[Path, str], - fromTensile: bool + cmdlineArchs: List[str] ) -> List[str]: """Compiles a HIP source code file into a code object file. @@ -190,25 +98,20 @@ def buildSourceCodeObjectFiles( destDir = Path(ensurePath(destDir)) kernelPath = Path(kernelPath) - if "CmakeCxxCompiler" in globalParameters and globalParameters["CmakeCxxCompiler"] is not None: - os.environ["CMAKE_CXX_COMPILER"] = globalParameters["CmakeCxxCompiler"] - objFilename = kernelPath.stem + '.o' coPathsRaw = [] coPaths= [] - _, cmdlineArchs = splitArchs(globalParameters, fromTensile) - objPath = str(tmpObjDir / objFilename) - toolchain.compile(str(kernelPath), objPath, str(includeDir), cmdlineArchs) + compiler(str(includeDir), cmdlineArchs, str(kernelPath), objPath) - for target in toolchain.targets(objPath): + for target in bundler.targets(objPath): match = re.search("gfx.*$", target) if match: arch = re.sub(":", "-", match.group()) coPathRaw = _computeSourceCodeObjectFilename(target, kernelPath.stem, tmpObjDir, arch) if not coPathRaw: continue - toolchain.unbundle(target, objPath, str(coPathRaw)) + bundler(target, objPath, str(coPathRaw)) coPath = str(destDir / coPathRaw.stem) coPathsRaw.append(coPathRaw) diff --git a/tensilelite/Tensile/Toolchain/Validators.py b/tensilelite/Tensile/Toolchain/Validators.py index 5c2381a652..7920fa96f4 100644 --- a/tensilelite/Tensile/Toolchain/Validators.py +++ b/tensilelite/Tensile/Toolchain/Validators.py @@ -246,19 +246,3 @@ def validateToolchain(*args: str): out = (_validateExecutable(x, searchPaths) for x in args) return next(out) if len(args) == 1 else tuple(out) - - -def getVersion(executable: str, versionFlag: str="--version", regex: str=r"version\s+([\d.]+)") -> str: - """Print the version of a toolchain component. - - Args: - executable: The toolchain component to check the version of. - versionFlag: The flag to pass to the executable to get the version. - """ - args = f'"{executable}" "{versionFlag}"' - try: - output = run(args, stdout=PIPE, shell=True).stdout.decode().strip() - match = re.search(regex, output, re.IGNORECASE) - return match.group(1) if match else "" - except Exception as e: - raise RuntimeError(f"Failed to get version when calling {args}: {e}") diff --git a/utilities/find_exact.py b/utilities/find_exact.py index 6b56b4b6a1..ff8dc5be35 100644 --- a/utilities/find_exact.py +++ b/utilities/find_exact.py @@ -1,6 +1,6 @@ ################################################################################ # -# Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal